{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import math, random\n",
    "\n",
    "import gym\n",
    "import numpy as np\n",
    "\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "import torch.autograd as autograd \n",
    "import torch.nn.functional as F"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from IPython.display import clear_output\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h3> If havs Cuda</h3>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "USE_CUDA = torch.cuda.is_available()\n",
    "Variable = lambda *args, **kwargs: autograd.Variable(*args, **kwargs).cuda() if USE_CUDA else autograd.Variable(*args, **kwargs)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h2>Replay Buffer</h2>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "from collections import deque\n",
    "\n",
    "class ReplayBuffer(object):\n",
    "    def __init__(self, capacity):\n",
    "        self.buffer = deque(maxlen=capacity)\n",
    "    \n",
    "    def push(self, state, action, reward, next_state, done):\n",
    "        state      = np.expand_dims(state, 0)\n",
    "        next_state = np.expand_dims(next_state, 0)\n",
    "            \n",
    "        self.buffer.append((state, action, reward, next_state, done))\n",
    "    \n",
    "    def sample(self, batch_size):\n",
    "        state, action, reward, next_state, done = zip(*random.sample(self.buffer, batch_size))\n",
    "        return np.concatenate(state), action, reward, np.concatenate(next_state), done\n",
    "    \n",
    "    def __len__(self):\n",
    "        return len(self.buffer)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h2>Cart Pole Environment</h2>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "env_id = \"CartPole-v0\"\n",
    "env = gym.make(env_id)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h2>Epsilon greedy exploration</h2>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "epsilon_start = 1.0\n",
    "epsilon_final = 0.01\n",
    "epsilon_decay = 500\n",
    "\n",
    "epsilon_by_frame = lambda frame_idx: epsilon_final + (epsilon_start - epsilon_final) * math.exp(-1. * frame_idx / epsilon_decay)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x12ed2c610>]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8li6FKAAAZ9UlEQVR4nO3dfXQd9X3n8ff3XulK1pP1aNmWbWQb24lJAgaFmIeT0IQQYFvc3W0b3GVDSFL6RFua7u6Bk57Qsv80TdvdZuOS0DakTSgOoWniEojbJpA2FFMLHMAPGGSDbQkbyY+yLcuypO/+MSNzLWTr2rrSaGY+r3Pu0cxvRvd+RyN/PPrNb2bM3RERkfjLRF2AiIgUhwJdRCQhFOgiIgmhQBcRSQgFuohIQpRE9cGNjY3e2toa1ceLiMTS888/v9/dm8ZaFlmgt7a20t7eHtXHi4jEkpntOtsydbmIiCSEAl1EJCEU6CIiCaFAFxFJCAW6iEhCjBvoZvY1M+s2s81nWW5m9iUz6zCzl8zs8uKXKSIi4ynkCP3rwI3nWH4TsCR83Qk8MPGyRETkfI0b6O7+r8DBc6yyCvhbD2wAas1sTrEKHG3jGwf5wg9eQbf9FRE5UzH60FuAPXnznWHbO5jZnWbWbmbtPT09F/RhL+45zANP76D3xOAFfb+ISFJN6UlRd3/Q3dvcva2pacwrV8fVUJUD4MDxk8UsTUQk9ooR6F3A/Lz5eWHbpKivLAPg4PGByfoIEZFYKkagrwM+EY52WQkccfe9RXjfMTVUjhyhK9BFRPKNe3MuM3sEuA5oNLNO4D6gFMDdvwI8AdwMdAB9wB2TVSxAfRjoOkIXETnTuIHu7qvHWe7AbxatonEo0EVExha7K0XLS7NU5rIcOKZAFxHJF7tAB6irzHFQo1xERM4Qy0BvqMzppKiIyCixDPT6ypz60EVERolpoJcp0EVERolloDdUBUfoup+LiMjbYhno9ZU5Tg4O0zcwFHUpIiLTRmwDHTQWXUQkXywDXZf/i4i8UywD/e0jdI1FFxEZEctAbwjvuKirRUVE3hbLQK+vUh+6iMhosQz0ylyWXElGgS4ikieWgW5muvxfRGSUWAY66PJ/EZHRYh3oOkIXEXlbbAO9QbfQFRE5Q3wDvapMwxZFRPLENtAbq8roGxiib2Aw6lJERKaF2AZ6U3VwcdH+ozpKFxGBBAR6z7H+iCsREZkeYhvojeHVoj1HdWJURARiHOinj9AV6CIiQIwDvaGyjIxBj0a6iIgAMQ70bMaoryzTEbqISCi2gQ5BP7oCXUQkEOtAb6ouo+eYAl1EBBIQ6Pt1hC4iAsQ90KuCI3R3j7oUEZHIxTvQq8sYGBymt1+X/4uIxD7QAfarH11EpLBAN7MbzWy7mXWY2T1jLF9gZk+Z2SYze8nMbi5+qe/UVKWLi0RERowb6GaWBdYANwHLgdVmtnzUar8PPOruK4Bbgb8odqFjadTVoiIipxVyhH4l0OHuO919AFgLrBq1jgM14fRM4M3ilXh2OkIXEXlbIYHeAuzJm+8M2/L9AXCbmXUCTwC/NdYbmdmdZtZuZu09PT0XUO6ZZs4opTRr6kMXEaF4J0VXA19393nAzcA3zOwd7+3uD7p7m7u3NTU1TfhDMxmjQZf/i4gAhQV6FzA/b35e2Jbv08CjAO7+LFAONBajwPHoalERkUAhgb4RWGJmC80sR3DSc92odXYDHwEws3cTBPrE+1QK0FRdRnevAl1EZNxAd/dB4C5gPbCNYDTLFjO738xuCVf7PeBXzOxF4BHgkz5Fl28215TRrS4XERFKClnJ3Z8gONmZ3/b5vOmtwDXFLa0wzTXlHDh+klNDw5RmY32dlIjIhMQ+AZtrynFHR+kiknqxD/TZNeUA7Duih0WLSLrFPtCbw0B/q1eBLiLpFvtAnz1TR+giIpCAQK+rKCVXktERuoikXuwD3cxoriljnwJdRFIu9oEOwYlRHaGLSNolItCba8p5S1eLikjKJSLQZ9eUs+9Iv54tKiKplohAb64p58SpIT1bVERSLRmBPlNj0UVEEhHoulpURCRpga4jdBFJsUQE+qya4Nmi3Qp0EUmxRAR6eWmWuopSHaGLSKolItAhGOmiPnQRSbPEBHpL7QzePKxAF5H0Skygz62dQdfhE1GXISISmUQF+pETpzh2UhcXiUg6JSbQW+pmALBXR+kiklLJCfTaYCx6pwJdRFIqMYE+tzY4Qn9TgS4iKZWYQJ9VXU5Jxug6pEAXkXRKTKBnM8bsmeU6QheR1EpMoIPGootIuiUu0DUWXUTSKlGBPrd2Bvt6+xkcGo66FBGRKZeoQG+pm8HQsPPWUT1fVETSJ1GBrqGLIpJmiQr0kYuLNHRRRNIoUYE+coSuE6MikkYFBbqZ3Whm282sw8zuOcs6v2RmW81si5n9XXHLLExFroS6ilI6dYQuIilUMt4KZpYF1gAfBTqBjWa2zt235q2zBLgXuMbdD5nZrMkqeDwL6ivoPNQX1ceLiESmkCP0K4EOd9/p7gPAWmDVqHV+BVjj7ocA3L27uGUWbn59BbsPKtBFJH0KCfQWYE/efGfYlm8psNTMnjGzDWZ241hvZGZ3mlm7mbX39PRcWMXjuKihgq5DJzQWXURSp1gnRUuAJcB1wGrgL82sdvRK7v6gu7e5e1tTU1ORPvpMC+orGBx29ur5oiKSMoUEehcwP29+XtiWrxNY5+6n3P114FWCgJ9yC+orAdh1QN0uIpIuhQT6RmCJmS00sxxwK7Bu1DrfJTg6x8waCbpgdhaxzoItaKgAUD+6iKTOuIHu7oPAXcB6YBvwqLtvMbP7zeyWcLX1wAEz2wo8BfxPdz8wWUWfy+yacnLZDLsOHo/i40VEIjPusEUAd38CeGJU2+fzph34bPiKVDZjzKubwR4doYtIyiTqStERCxo0dFFE0ieZgV5fwa4DfQR/OIiIpENiA/1o/yBHTpyKuhQRkSmT2EAHDV0UkXRJZqBr6KKIpFAyA/30EbqGLopIeiQy0CtyJcyuKWfnfgW6iKRHIgMdYFFTJTt7FOgikh6JDfSFjZXs7DmmoYsikhqJDfRFTVX09g9y4PhA1KWIiEyJBAd6cNdFdbuISFokNtAXN1YB8Pr+YxFXIiIyNRIb6C11M8iVZHSELiKpkdhAz2aM1oYKdijQRSQlEhvoAIsaq9ipLhcRSYlkB3pTJbsP9HFKD4wWkRRIeKBXMTjsetiFiKRCwgNdQxdFJD0SHeiLm4Khi691qx9dRJIv0YE+c0Ypc2aW8+pbR6MuRURk0iU60AGWNlezfZ8CXUSSL/GBvmx2NR09xxjUSBcRSbjkB3pzNQODw+zSSBcRSbjkB/rsagB1u4hI4iU+0C+eVYWZAl1Eki/xgV5emqW1oVIjXUQk8RIf6ABLm6vYrkAXkYRLRaAva67mjf3H6T81FHUpIiKTJhWBvnR2NcMOHbpiVEQSLBWB/u45NQBs29sbcSUiIpMnFYG+sKGSylyWLW8q0EUkuQoKdDO70cy2m1mHmd1zjvX+q5m5mbUVr8SJy2SM5XNr2Nx1JOpSREQmzbiBbmZZYA1wE7AcWG1my8dYrxr4HeC5YhdZDJfMncnWvb0MDXvUpYiITIpCjtCvBDrcfae7DwBrgVVjrPe/gS8A/UWsr2je0zKTvoEhXt+ve6OLSDIVEugtwJ68+c6w7TQzuxyY7+7fP9cbmdmdZtZuZu09PT3nXexEvKclODG65U11u4hIMk34pKiZZYA/A35vvHXd/UF3b3P3tqampol+9HlZ3FRFriSjfnQRSaxCAr0LmJ83Py9sG1ENvAd42szeAFYC66bbidHSbIZ3z65mc5dGuohIMhUS6BuBJWa20MxywK3AupGF7n7E3RvdvdXdW4ENwC3u3j4pFU/AJS0z2fzmEdx1YlREkmfcQHf3QeAuYD2wDXjU3beY2f1mdstkF1hM75k7k6P9g+zWvdFFJIFKClnJ3Z8AnhjV9vmzrHvdxMuaHJfNrwXgp3sOc1FDZcTViIgUVyquFB2xtLmKilyWF3YdiroUEZGiS1Wgl2QzXDqvlhd2H466FBGRoktVoAOsWFDLtr29nBjQrXRFJFlSF+iXL6hjcNh5WePRRSRhUhfoly0IToxu2q1+dBFJltQFemNVGRc1VPCCAl1EEiZ1gQ6wYn5wYlQXGIlIkqQy0K+4qI6eoyfZc/BE1KWIiBRNKgN95aIGADbsPBBxJSIixZPKQL94VhWNVTmeVaCLSIKkMtDNjA8samDDzgPqRxeRxEhloEPQ7bL3SL9u1CUiiZHaQL9qUT2gfnQRSY7UBvripqAffcPOg1GXIiJSFKkN9JF+9Gd3qB9dRJIhtYEOcO3Fjezr7ee17mNRlyIiMmGpDvQPLQ0eVP3j7T0RVyIiMnGpDvS5tTNY2lzF0692R12KiMiEpTrQAa5bNouNrx/i+MnBqEsREZmQ1Af6h5Y2MTA0zLM7NHxRROIt9YHe1lpHRS6rbhcRib3UB3pZSZarFzfw1Cs9Gr4oIrGW+kAHuGH5bLoOn2DLm71RlyIicsEU6MD1y5vJGPxg876oSxERuWAKdKC+MscHFjbw5Oa9UZciInLBFOihm947mx09x+noPhp1KSIiF0SBHrph+WwAnnxZ3S4iEk8K9NDsmeWsWFDL919Wt4uIxJMCPc/PX9bCK/uOsm2vRruISPwo0PP83KVzKckY/7CpK+pSRETOmwI9T31ljuuWzeK7m7oYGtZFRiISLwUFupndaGbbzazDzO4ZY/lnzWyrmb1kZj80s4uKX+rU+C+Xt9B99CTPdOyPuhQRkfMybqCbWRZYA9wELAdWm9nyUattAtrc/X3AY8AfF7vQqfLhd82ipryEx57vjLoUEZHzUsgR+pVAh7vvdPcBYC2wKn8Fd3/K3fvC2Q3AvOKWOXXKS7P85xUt/GDzPg4cOxl1OSIiBSsk0FuAPXnznWHb2XwaeHKsBWZ2p5m1m1l7T8/0fUrQbSsvYmBomEfbdZQuIvFR1JOiZnYb0AZ8cazl7v6gu7e5e1tTU1MxP7qoljRXs3JRPQ8/t0snR0UkNgoJ9C5gft78vLDtDGZ2PfA54BZ3j31fxX9f2UrnoRP8WPdJF5GYKCTQNwJLzGyhmeWAW4F1+SuY2QrgqwRhnogEvOGSZmZVl/HQM29EXYqISEHGDXR3HwTuAtYD24BH3X2Lmd1vZreEq30RqAK+bWY/NbN1Z3m72CjNZvjUtQv5t9f283LnkajLEREZl0X1lJ62tjZvb2+P5LMLdbT/FFf/0Y+49uJGHrjtiqjLERHBzJ5397axlulK0XOoLi/lE1ddxA+27KOj+1jU5YiInJMCfRx3XLOQXDbDXzzdEXUpIiLnpEAfR2NVGbdf3co/bOpi+z49/EJEpi8FegF+47rFVJWV8MX1r0RdiojIWSnQC1BbkePXPrSYf9nWzcY3DkZdjojImBToBfrUNQtprinjD/9xi64eFZFpSYFeoBm5LL//n5azuauXb27YFXU5IiLvoEA/Dz/7vjlce3Ejf7J+O929/VGXIyJyBgX6eTAz7l91CScHh7lv3RaiuihLRGQsCvTztKipirs/uoQnN+/jOy/o2aMiMn0o0C/Ar35wMVe21nPfui3sOdg3/jeIiEwBBfoFyGaMP/2lSwH47bWbODk4FHFFIiIK9As2v76CL/7C+9i0+zD3fU/96SISPQX6BNz03jn85s8sZu3GPXzzud1RlyMiKVcSdQFx99mPLmPb3qPc973NzKou42OXzI66JBFJKR2hT1A2Y3z5l1dw6fxafuuRTfz7jv1RlyQiKaVAL4KKXAkPffL9tDZU8Jm/aeeZDoW6iEw9BXqR1Fbk+OZnPsD8ugrueGgj/7RlX9QliUjKKNCLaFZ1Od/61ZUsn1vDrz/8Al/7yesa/SIiU0aBXmS1FTke/swH+Mi7ZnH/41v5H99+if5TGqcuIpNPgT4JKstK+MptV3D39Uv4+xc6WfXlZ9jy5pGoyxKRhFOgT5JMxrj7+qU8dMf7Odg3wM+veYb/98PXdFWpiEwaBfok+5lls/inuz/IDZfM5k//+VU+9n/+lR+98lbUZYlIAinQp0BdZY41v3w5X7/j/WQyxqe+3s7qBzfw7I4DUZcmIgliUY3CaGtr8/b29kg+O0oDg8N8c8MuHvjxDnqOnuTK1nruuKaV65c3U5rV/68icm5m9ry7t425TIEejf5TQ6z9j9385b+9TtfhE8yqLuPj75/PqsvmcvGs6qjLE5FpSoE+jQ0NO0+90s3Dz+3i6Vd7cIdlzdXc/N45fPhds7hkbg2ZjEVdpohMEwr0mHirt58nX97L91/eS/uuQ7hDXUUpVy9u5KrFDVw2v5Zls6vVNSOSYgr0GOo+2s+/dxzgJx37+clr+9kXPpQ6V5Lhkrk1vLdlJhfPqmJxU/BqrinDTEfyIkmnQI85d2fPwRO82HmYlzoP8+KeI2zd28uxk4On16kqK2Fe3Qzm1s5gbm05c2YGX5ury6mrzFFfmaO2opSykmyEWyIiE3WuQNf90GPAzFjQUMGChgp+7tK5QBDy3UdPsqP7GB09x9jRfYzOQyd480g/L+w+xOG+U2O+V1VZCXWVpdRV5KjMlVBZVkJlWTb4mhv5WkJFWZaykiylWaOsJEOuJEMuG8znwvmykgyl2eCVzRgZM7IZI2tGJsPptpH2jKG/IkQmUUGBbmY3An8OZIG/cvc/GrW8DPhb4ArgAPBxd3+juKVKPjOjuaac5ppyrr648R3L+wYG2Xukn7d6+zncd4qDxwc4dHyAQ32nONQ3wMHjAxw/OUjX4RP0DQxy/OQgx08OcWKS7zuTMd4R/hYG/UjWW7h9I9EftNvp6fx2G7Pd8r7v3OtNu/9epllB06ycaXdAcKHV/PZHlpw+OCumcQPdzLLAGuCjQCew0czWufvWvNU+DRxy94vN7FbgC8DHi16tFKwiV3K6f/18DA07fQOD9A0MMTA4zMnBYU4NDTMwOMzAyNdR7aeGhhlyZ3jYGRp2hpxg2oN5d2domLfXOWNdx53Td6V0COYJ5x1GOgWDVfLawwWO502f+f2c8f1+xntNt/tgTrc7c06vaph2BfkECpo5o7SIlbytkCP0K4EOd98JYGZrgVVAfqCvAv4gnH4M+LKZmU+331AZVzZjVJeXUl0+Ob9wIjJ5Chn/1gLsyZvvDNvGXMfdB4EjQMPoNzKzO82s3czae3p6LqxiEREZ05QOaHb3B929zd3bmpqapvKjRUQSr5BA7wLm583PC9vGXMfMSoCZBCdHRURkihQS6BuBJWa20MxywK3AulHrrANuD6d/AfiR+s9FRKbWuCdF3X3QzO4C1hMMW/yau28xs/uBdndfB/w18A0z6wAOEoS+iIhMoYLGobv7E8ATo9o+nzfdD/xicUsTEZHzobs8iYgkhAJdRCQhIrs5l5n1ALsu8Nsbgf1FLCcOtM3poG1Oh4ls80XuPua478gCfSLMrP1sdxtLKm1zOmib02GytlldLiIiCaFAFxFJiLgG+oNRFxABbXM6aJvTYVK2OZZ96CIi8k5xPUIXEZFRFOgiIgkRu0A3sxvNbLuZdZjZPVHXc6HMbL6ZPWVmW81si5n9Ttheb2b/bGavhV/rwnYzsy+F2/2SmV2e9163h+u/Zma3n+0zpwszy5rZJjN7PJxfaGbPhdv2rfAmcJhZWTjfES5vzXuPe8P27Wb2sWi2pDBmVmtmj5nZK2a2zcyuSvp+NrPfDX+vN5vZI2ZWnrT9bGZfM7NuM9uc11a0/WpmV5jZy+H3fMmsgOfvuXtsXgQ3B9sBLAJywIvA8qjrusBtmQNcHk5XA68Cy4E/Bu4J2+8BvhBO3ww8SfAYw5XAc2F7PbAz/FoXTtdFvX3jbPtngb8DHg/nHwVuDae/Avx6OP0bwFfC6VuBb4XTy8N9XwYsDH8nslFv1zm292+Az4TTOaA2yfuZ4IE3rwMz8vbvJ5O2n4EPApcDm/PairZfgf8I17Xwe28at6aofyjn+QO8ClifN38vcG/UdRVp275H8NzW7cCcsG0OsD2c/iqwOm/97eHy1cBX89rPWG+6vQjup/9D4MPA4+Ev636gZPQ+JrjD51XhdEm4no3e7/nrTbcXwbMBXiccgDB6/yVxP/P2E8zqw/32OPCxJO5noHVUoBdlv4bLXslrP2O9s73i1uVSyOPwYif8E3MF8BzQ7O57w0X7gOZw+mzbHrefyf8F/hcwHM43AIc9eHQhnFn/2R5tGKdtXgj0AA+F3Ux/ZWaVJHg/u3sX8CfAbmAvwX57nmTv5xHF2q8t4fTo9nOKW6AnjplVAX8P3O3uvfnLPPivOTHjSs3sZ4Fud38+6lqmUAnBn+UPuPsK4DjBn+KnJXA/1xE8OH4hMBeoBG6MtKgIRLFf4xbohTwOLzbMrJQgzB929++EzW+Z2Zxw+RygO2w/27bH6WdyDXCLmb0BrCXodvlzoNaCRxfCmfWf7dGGcdrmTqDT3Z8L5x8jCPgk7+frgdfdvcfdTwHfIdj3Sd7PI4q1X7vC6dHt5xS3QC/kcXixEJ6x/mtgm7v/Wd6i/Mf53U7Qtz7S/onwbPlK4Ej4p9164AYzqwuPjG4I26Ydd7/X3ee5eyvBvvuRu/834CmCRxfCO7d5rEcbrgNuDUdHLASWEJxAmnbcfR+wx8yWhU0fAbaS4P1M0NWy0swqwt/zkW1O7H7OU5T9Gi7rNbOV4c/wE3nvdXZRn1S4gJMQNxOMCNkBfC7qeiawHdcS/Dn2EvDT8HUzQd/hD4HXgH8B6sP1DVgTbvfLQFvee30K6Ahfd0S9bQVu/3W8PcplEcE/1A7g20BZ2F4ezneEyxflff/nwp/Fdgo4+x/xtl4GtIf7+rsEoxkSvZ+BPwReATYD3yAYqZKo/Qw8QnCO4BTBX2KfLuZ+BdrCn98O4MuMOrE+1kuX/ouIJETculxEROQsFOgiIgmhQBcRSQgFuohIQijQRUQSQoEuIpIQCnQRkYT4/3B4SeNAOVqBAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot([epsilon_by_frame(i) for i in range(10000)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.010044945930464861"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "epsilon_by_frame(5000)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h2>Deep Q Network</h2>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "class DQN(nn.Module):\n",
    "    def __init__(self, num_inputs, num_actions):\n",
    "        super(DQN, self).__init__()\n",
    "        \n",
    "        self.layers = nn.Sequential(\n",
    "            nn.Linear(env.observation_space.shape[0], 128),\n",
    "            nn.ReLU(),\n",
    "            nn.Linear(128, 128),\n",
    "            nn.ReLU(),\n",
    "            nn.Linear(128, env.action_space.n)\n",
    "        )\n",
    "        \n",
    "    def forward(self, x):\n",
    "        return self.layers(x)\n",
    "    \n",
    "    def act(self, state, epsilon):\n",
    "        if random.random() > epsilon:\n",
    "            state   = Variable(torch.FloatTensor(state).unsqueeze(0), volatile=True)\n",
    "            q_value = self.forward(state)\n",
    "            action  = q_value.max(1)[1].data[0].item()\n",
    "        else:\n",
    "            action = random.randrange(env.action_space.n)\n",
    "        return action"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = DQN(env.observation_space.shape[0], env.action_space.n)\n",
    "\n",
    "if USE_CUDA:\n",
    "    model = model.cuda()\n",
    "    \n",
    "optimizer = optim.Adam(model.parameters())\n",
    "\n",
    "replay_buffer = ReplayBuffer(1000)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h2>Computing Temporal Difference Loss</h2>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def compute_td_loss(batch_size):\n",
    "    state, action, reward, next_state, done = replay_buffer.sample(batch_size)\n",
    "\n",
    "    state      = Variable(torch.FloatTensor(np.float32(state)))\n",
    "    next_state = Variable(torch.FloatTensor(np.float32(next_state)), volatile=True)\n",
    "    action     = Variable(torch.LongTensor(action))\n",
    "    reward     = Variable(torch.FloatTensor(reward))\n",
    "    done       = Variable(torch.FloatTensor(done))\n",
    "\n",
    "    q_values      = model(state)\n",
    "    next_q_values = model(next_state)\n",
    "\n",
    "    q_value          = q_values.gather(1, action.unsqueeze(1)).squeeze(1)\n",
    "    next_q_value     = next_q_values.max(1)[0]\n",
    "    expected_q_value = reward + gamma * next_q_value * (1 - done)\n",
    "    \n",
    "    loss = (q_value - Variable(expected_q_value.data)).pow(2).mean()\n",
    "        \n",
    "    optimizer.zero_grad()\n",
    "    loss.backward()\n",
    "    optimizer.step()\n",
    "    \n",
    "    return loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot(frame_idx, num_frames, rewards, losses):\n",
    "    clear_output(True)\n",
    "    plt.figure(figsize=(20,5))\n",
    "    plt.subplot(131)\n",
    "    plt.title('frame %s / %s. reward: %s' % (frame_idx, num_frames, np.mean(rewards[-10:])))\n",
    "    plt.plot(rewards)\n",
    "    plt.subplot(132)\n",
    "    plt.title('loss')\n",
    "    plt.plot(losses)\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h2>Training</h2>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 135,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAvwAAAE/CAYAAAA6zBcIAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8li6FKAAAgAElEQVR4nOzdeZhkZ3nf/e9dW++zj0Y7IwmBLQgIPCyOzWIEWIBj4cTGYGJkh1jxa0jswHvZInaCAybGTmwcvMALQUHYBkzAGIUXB2TMbgRIWAgJARoJiZmRNNOz9l7LOU/+OM+pPl1dW1d11Tld/ftcV1/TfWo7PdPTddddv+d+zDmHiIiIiIiMplzaJyAiIiIiIoOjgl9EREREZISp4BcRERERGWEq+EVERERERpgKfhERERGREaaCX0RERERkhKngFxEREQHM7EEze37a5yGy2VTwD4GZPd7M7jSzeTP7d2mfj0gnZvbbZvYXaZ+HiIiI9E8F/3D8OvAZ59yMc+7taZ9MIzN7l5l9x8xCM/uFJpf/ezN71MzmzOwmMxtLXHbQzD5jZktm9u3Gzkg/t21xrt8xs8c1Of4yM/sHf1+fbXL51WZ2h7/8DjO7OnGZmdnvmdkp//F7ZmabcdvtwMxKZvZh3xlzZvbchsvHzOydZnbczE6b2f82s4sSl73HzB7yL4jvNLMXdXi8lj9TIiIisp4K/uF4DHBPqwvNLD/Ec2nmG8CvAF9vvMDMfhy4EbiG6Pu4HPjPiat8APhHYC/wm8CHzWx/v7dtxsyuAPLOue82ufg08EfAW5vcrgR8DPgLYDdwM/AxfxzgBuClwJOBJwH/DPg3/d52o8ys0Mvt+uFfsGzG74EvAv8SeLTJZb8K/DDR38+FwBngj/1lBeAI8BxgJ/BbwIfM7GCL8+30MyUi0jffjPgjM3vYf/xR3Fwws31m9nEzO+ubGF+If4+a2W+Y2THfwPiOmV2T7nci4jnn9DHAD+DvgQBYARaAxwHvBd4BfAJYBJ4PvISo+J0jKoB+O3EfBwEH/KK/7Azwy8DTgLuAs8CfNDzuvwLu9df9JPCYLs71i8AvNBx7P/BfEl9fAzzqP38cUAZmEpd/Afjlfm/b4vz+HfD2Dt/DvwY+23DshcAxwBLHvg9c6z//B+CGxGWvBm7r97Zd/H3/NvBhohcTc/7cc0QF7f3AKeBDwB5//ZuB1/vPL/I/E6/xX19B9KInR/TC5OPArP/3/zhwceJxPwu8BfgSsAw8FrgM+BwwD9wK/AnwFz38vB8Fnttw7B3A7ye+fgnwnTb3cRfwL1pc1vJnSh/60Ic++v0AHiR6Tn4TcBtwHrDf/65/s7/O7wLvBIr+41mAAY8neo6+0F/vIHBF2t+TPvThnFOHf9Ccc88jKmRf65ybdqvd6Z8jKrpmiArtReBVwC6iguj/MbOXNtzdM4ArgZ8l6mb/JtEvpicALzOz5wCY2XXAfwD+OdEvqi8QddN78QSidwBi3wAOmNlef9kDzrn5hsufsAm3bebFwP/f4/dwl3POJY7d1eE8n7AJt+3GdURF/y7gL4F/S/SOwXNY7Yb/qb/u54Dn+s+fAzwAPDvx9ReccyFR0f8/iTrglxIV9X/S8Lg/T/TuxAzwEFEhfQewD3gzcH3yymZ2l5n93Aa+r6T3AD9iZhea2STwSuBvm13RzA4QvRhs9Y5Yu58pEZHN8krgTc65E865WaJ3En/eX1YFLiBqpFWdc1/wzxEBMAZcZWZF59yDzrn7Uzl7kQYq+NPzMefcl5xzoXNuxTn3WefcN/3XdxEV6M9puM2b/XU/RfQC4QP+l9ExoqL+Kf56vwz8rnPuXudcDfgvwNVm9pgeznMaOJf4Ov58psll8eUzm3DbNXyh+DSi7vRG9XKe0z6L389tu/Fl59zf+H/3ZaJ/u990zh11zpWJ3gX4aR/3+Rzwo/6t42cDvw/8iL+f5/jLcc6dcs59xDm35F9QvYX1P0vvdc7d438+LiD6u/2Pzrmyc+7zwP9OXtk59yTn3Pu7/J4a3UfU9TpG9E7GDxJ1z9YwsyLRi56bnXPfbnFf7X6mREQ2y4VEzZDYQ/4YwH8FDgOfMrMHzOxGAOfcYeDXiH5vnzCzD5rZhYhkgAr+9BxJfmFmz/ALWGfN7BxR4bev4TbHE58vN/l62n/+GOC/+3zhWaKohxHFQDZqAdiR+Dr+fL7JZfHlcde+n9s2ugb4B18Eb1Qv57ngOzb93LYbRxq+fgzw0cS/3b1EXaMDvlO0CFxN9Bbyx4GHzezxJAp+M5s0s//PL4SdAz4P7GpYK5J83AuBM865xcSx5BNdv/6UqOu1F5gC/pqGDr9/EfPnQAV4bZv7avczJSKyWR4m+n0cu9Qfwzk375x7vXPucuAngdfFWX3n3Pudcz/qb+uA3xvuaYs0p4I/PY0F4fuBW4BLnHM7ifKBvU57OQL8G+fcrsTHhHPuH3q4r3uIFqTGngwcd86d8pddbmYzDZffswm3bfRiojUPvbgHeFJD1/1JHc7znk24bTcafw6OAC9q+Lcb9+/iQFTU/zRQ8sc+RxS/2Q3c6a/zeqIs6TOccztYjf0kv4fk4z4C7DazqcSxSzfwPXRyNdE7Cqf9C7Y/Bp5uZvsgWjhMFPs5QJTdr7a5r3Y/UyIim+UDwG+Z2X7/u+o/Ea23wsx+wswe6393nSNqyoQWjeB+nl/cu0LUiAtTOn+RNVTwZ8cMcNo5t2JmTyfK+PfqncAbzOwJAGa208x+ptWV/VjFcaKCsGhm44nJLe8DXm1mV5nZLqIpKu8F8OsR7gTe6G/zU0TF8Ec24baNXkSb/L6Z5f33UABy/j6L/uLPEv1C/nd+8kLcQf77xHm+zswu8m+/vj4+zz5v24t3Am+J41f+yea6xOWfI+qAfz5xfq8FvuicC/yxGaInmrNmtgd4Y7sHdM49BNwO/Gf/s/CjRNOGuub/bsb9lyX/9x+/wPga8Cr/c1gkmgj1sHPupL/8HUQxn3/mY03ttPyZEhHZRL9D9HvxLuCbRFPsfsdfdiXwd0TvOH4Z+DPn3GeI3sl8K3CSaGLZecAbhnvaIi2kvWp4O3wQFWX/OvH1e4HfabjOTxPFKOaJohr1KSmsTukpJK6/ZhoKUefhtxJf/zzRL6l46s9NHc7PNXwk7/t1RPGhOaLFoGOJyw762y8D3wGe33DfPd82cb0nAnd3+Dv+hSbfw3sTlz+FaFHqMtEv7qckLjOiPPxp//H7rJ3K089tF4BntTjn36ZhEg7Ri/DX+b+PeaJpPcmpNI/339v1/uudQA34jcR1LvR/rwvAd4nGhNZ/fmj4efTHLidaB7JAkyk9RJ31V7b5+3+wyd//QX/ZXqJs/gmiiVJfBJ7uL4vf9o6nWMUfr/SXX+q/vrSbnyl96EMf+tCHPvSx/sOc6zZqLJIOM/t1YJ9z7tfTPhcRERGRrWboG/2I9OBBGqbGiIiIiEh31OEXERERERlhWrQrIiIiIjLCVPCLiIiIiIywTGT49+3b5w4ePJj2aYiIZNIdd9xx0jm3P+3zSJOeJ0REmuvmOSITBf/Bgwe5/fbb0z4NEZFMMrPN3Pl4S9LzhIhIc908RyjSIyIiIiIywlTwi4iIiIiMMBX8IiIiIiIjTAW/iIiIiMgIU8EvIiIiIjLCVPCLiIiIiIwwFfwiIiIiIiOsY8FvZpeY2WfM7Ftmdo+Z/ao/vsfMbjWz+/yfu/1xM7O3m9lhM7vLzJ466G9CRERERESa66bDXwNe75y7Cngm8Bozuwq4Efi0c+5K4NP+a4AXAVf6jxuAd2z6WYuIiIiISFc67rTrnHsEeMR/Pm9m9wIXAdcBz/VXuxn4LPAb/vj7nHMOuM3MdpnZBf5+ZJuanS9zfG6FJ160s+vb/MP9J3nawT0U89lJnn3z6DnufvhcV9d97uP3c8HOiXXHHzm3zOe/O0vo1h6/ePcEz7qy7c7YsgG3P3ia+04sDO3xrjxvmkMH9wzt8URk6zh8YoHxYo6Ld0+mfSqyTXUs+JPM7CDwFOArwIFEEf8ocMB/fhFwJHGzo/7YmoLfzG4gegeASy+9dIOnLVvNOz93P7d842G+9pvP7+r6R04v8XPv/grv/JdP5donXjDgs+ve6//XnXz3eHdF5EuedAF/+nOribaTC2X+7DP38xdfeYhKLVx3/ZzBvW++lrFCftPOd7tyzvGL7/0a8yu1oT3m3qkSd/zHFwzt8URk63j+H34OgAff+pKUz0S2q64LfjObBj4C/Jpzbs7M6pc555yZuZY3bsI59y7gXQCHDh3a0G1l65lfqXJ6sYJzjuTPTitLlQCAxXIw6FPbkKVKwIueeD5v/GdPaHu93/3be/nsd2YJQkc+ZwSh46f+7EscO7PMv3jqxfzSsy9nx3ixfv33f/X7vP3T91ENHGMbehkuzZxerDC/UuPfP/9x/OzTLhn4473js4f5wFePdL6iiIhICroqLcysSFTs/6Vz7q/94eNxVMfMLgBO+OPHgOQz7MX+mGxjlVpIEDqWKgFTXVS01SDqgAeNuZeUBaFjeqzA+TvH217v+T94gI/d+TDfOHqWp166m689eJojp5f5o5+9mpc+5aJ11989GRX/tWB951827siZZQB+8IKZjv9Wm2Hf9BiVIKQWhBQyFEETERGB7qb0GPAe4F7n3B8mLroFuN5/fj3wscTxV/lpPc8Ezim/LxVfyHYbsYgL/VrGCv5a6Loq6H70sfswgy989yQA/+fuRxkr5HjBVQeaXj++z2qQre93qzpyegmAS/YMJy87UYpiWMvVbL0jJSIiAt1N6fkR4OeB55nZnf7jxcBbgReY2X3A8/3XAJ8AHgAOA+8GfmXzT1u2mjizPr9S7er6tTDu8Ger410LQgq5zpGk3VMlnnTRTj5/3yxh6PjkPY/y7Mftb/nuRnyftYx9v1vVkTPDLfjHiyr4RUQku7qZ0vNFoFWFc02T6zvgNX2el4yYsi/457rs8Med7ix2+PNdFPwAz37cfv7ss/fzxcMneeTcCv/vCx/f8rr1gl8d/k1x5PQyuyeLTA9pQcSEL/hXKnrBJiIi2aOwqQxFeYMd/jjSk8UMfzHffcEfhI43ffxbFHLG83+weZwHqI8erSrDvymOnlkaWncfFOkREZFsU8Evfbnn4XP8jy880PF6q5Gebjv82Vy0Wwsc+Vx3/22uvmQX02MFDp9Y4Iev2MvOyWLL6xb8i4isfb9b1ZHTS1wyxHnXE4r0iIhIhqngl77ccufDvPVvv93xehst+GuZjfR0l+GHqGv/T6/YC8C1Tzy/7XULOS3a3SxB6Dh2dpmL96zf9GxQ6hn+ytYq+M3sQTP7pl+bdbs/tsfMbjWz+/yfu/1xM7O3m9lhM7vLzJ6auJ/r/fXvM7PrE8d/yN//YX9ba/cYIiIyGCr4pS+VIKQWOqKlG+2vBxtZtJu9SE8YOkK32o3vxkuedAHTYwVeeFWngl+LdjfL8bkVqoEbboffR3pWtmaH/8ecc1c75w75r28EPu2cuxL4tP8a4EXAlf7jBuAdEBXvwBuBZwBPB96YKODfAfxS4nbXdngMEREZABX80pc4etOpE7/hDn/Y3f0OU3wu3Xb4AX7yyRdy+289n/0zY22vF7+IUIe/f8MeyQkjF+m5DrjZf34z8NLE8fe5yG3ALr8Hy48DtzrnTjvnzgC3Atf6y3Y4527zwxze13BfzR5DREQGQAW/9KVa664Tv9GxnKuLdrPT8Y7PqdsMP4CZ1eMe7cSLdrXxVv/iTbcu2T28SM/EFo30AA74lJndYWY3+GMHEnunPArEq80vApLbCR/1x9odP9rkeLvHEBGRARjOzDoZWXGHvxqEbQvbjW68lcWxnPG7Dt1O6dmI+F2DLEWYtqojp5cwg4uGWPCPl6IXbEtbr8P/o865Y2Z2HnCrma1ZkOOcc2Y20B/Kdo/hX4TcAHDppZcO8jREREaaOvzSl7iQ7zQ/vrLBOfxxpzvIUMQl/h67ncO/EfWddlXw9+3ImSUOzIwzVuj8zspmmSxFvZOVLdbhd84d83+eAD5KlME/7uM4+D9P+KsfAy5J3Pxif6zd8YubHKfNYzSe37ucc4ecc4f279/f67cpIrLtqeCXvtQ7/B2iNxvfaTeLHX6f4c9v/n+b1Y23FOnp19HTy1wyxAk9AOOF6GdiK2X4zWzKzGbiz4EXAncDtwDxpJ3rgY/5z28BXuWn9TwTOOdjOZ8EXmhmu/1i3RcCn/SXzZnZM/10nlc13FezxxAZOeeWunveExkkRXqkL/XoTZtOvHNuw5GeWgbn8Ac9LNrtlhbtbp4jZ5b44cv3DvUxC/kcpXxuSxX8RLn5j/pJmQXg/c65/2NmXwM+ZGavBh4CXuav/wngxcBhYAn4RQDn3GkzezPwNX+9NznnTvvPfwV4LzAB/K3/AHhri8cQGTn/6Za70z4FERX80p9uNsiqJLrW8+Wt2+GPv9dBRHrqi3YztEh5KyrXAh6dW+HiIU7oiY0Xc1tq0a5z7gHgyU2OnwKuaXLcAa9pcV83ATc1OX478MRuH0NkFC2Wt87vBRldivRIX+KoTrVNFCW+DmxkLGdU6IcZKvjjFzVatJtdD59dwbnhTuiJTZTyW3UOv4iIjDgV/NKXbubwxwX/zHiB+ZVax026YDXSk6UOf62HsZzdijv8ivT0J40Z/LGJYn6rRXpERGSbUMEvfYkL1HYd/rIv+PdNjxGErquiqJbBOfxx3GYQGf68Fu1uiqN+Bv/FKXT4x4v5LRXpERGR7UMFv/Sl2sVYzrjDv3eqBHQX66llcQ5/MIRFuxn6frei43MrmMGBHeNDf+yJkjr8IiKSTSr4pS+VbiI9/jp7p+OCv/PC3XjMZ5Yy7fUpPQPI8Bdz2ml3M8wulNkzWapHpIZpoqgMv4iIZJMKfunLaoe/86LdvdNjQHebbwVZ7PCH8ZSeAczhz2vR7mY4MVdm/8xYKo89UcyzpEiPiIhkkAp+6Uu11rkwr2f4NxLpqWf4s1MAx5Ge4gDHcmrRbn9m51dSK/jHFekREZGMUsEvfYm73t2M5Yw7/N1EeuL7zVKHP6hP6dGi3aw6MV/mvJnh5/cBJot5VtThFxGRDFLBL32Ji/m2i3bXZfi7X7SbpSk91QFm+OOFwFq027swdJxcKHPejpQiPerwi4hIRqngl75Uu8jar07p6b7DX7/fDEVcgvpYzs3/b2NmFHKmDn8fzi5XqQaO/dPpZfhV8IuISBap4Je+rG681TnSs2eqhFl3Hf64uA672KRrWOIXH4OI9ED0zkGW1ixsNSfmVwBS6/CPF/OsVMNM7Q4tIiICKvilD2Ho6p399pGeqOs5VsgxPVboquCPoy1ZyvDXBhjpgWg0pxbt9u7EXBkgtQz/RCkPrC5SFxERyYqOBb+Z3WRmJ8zs7sSxvzKzO/3Hg2Z2pz9+0MyWE5e9c5AnL+mqJrr63SzaLRVy7BgvMtfNot0ge3P46wX/ACI9APm8tX2nRNqbnY8K/jTHcgKK9YiISOYUurjOe4E/Ad4XH3DO/Wz8uZn9AXAucf37nXNXb9YJSnYlu9HdZPhLhRwz4911+IMu3jkYttUM/4AiPerw9+XEfNzhV8EvIiKS1LHgd8593swONrvMzAx4GfC8zT0t2QqqiehCN3P4Vwv+7hftZqnDXx1whr+Y16LdfpyYX2GqlGdqrJs+xuYb95Ge5UrnF7QiIiLD1G824VnAcefcfYljl5nZP5rZ58zsWX3ev2RYMsbTdqddf1kpn2NmvLixDn+GIi7xOcWbZG02Ldrtz+x8mfN2pJPfh0SHv5Kdn1kRERHov+B/BfCBxNePAJc6554CvA54v5ntaHZDM7vBzG43s9tnZ2f7PA1JQ2VNwd+mw19NFvxdLtrNcIZ/YB3+XE5z+PtwYr6c2khOUKRHRESyq+eC38wKwD8H/io+5pwrO+dO+c/vAO4HHtfs9s65dznnDjnnDu3fv7/X05AUJfPm1XZjOYOQYt7I5azrSE8ti1N6gsFm+POaw9+X2fky+1MayQkwUYp+nargFxGRrOmnw/984NvOuaPxATPbb2Z5//nlwJXAA/2domRVtcsOf6UWUvIxmDjS4zrM148L/Sx1+IMBj+Us5LVotx8n5lZSW7AL0Rx+gOWKCn4REcmWbsZyfgD4MvB4MztqZq/2F72ctXEegGcDd/kxnR8Gftk5d3ozT1iyo1LrMsNfCykV4oK/QC10rFTbd7K341jOosZy9myxXGOxEqQ2gx9gshQtFl5Rh19E1sjO85hsX91M6XlFi+O/0OTYR4CP9H9ashWs6fB3GMu5WvAXAZhfqdY3KmomyGCHP34RMrCddnNatNurtGfwgzL8IiKSXdppV3rW9Rz+YLXg3zEevcac67BwN34xkakMf73DP8hIjzr8vUh7Bj8kp/So4BcRkWxRwS89SxannXbaXc3wRwV/p4W7Wc3w5wxyA+zwZ2mjsa3kxPwKAOeluGh3XIt2RUQko1TwS8+6HstZCykVou7naqSnfYc/vr8sZdqrgRtYfh98hz9DL3C2ktl6hz+9DH8pnyNnyvCLiEj2qOCXnq3dabf9WM7kol3oouAPs7doNwjDgU3oAShqLGfPTsyXKeSMXRPF1M7BzJgo5llSpEdERDJGBb/0bE2Gv+1YzoCx/PpFu+2sdvizU/DXQjewBbsQjftUpKc3J+bK7J8ZG1jcqlsTpbwiPSIikjkq+KVnG5nSM1aMftTGfae/U+whvj/nIMxI0V8L3MAW7EIU6clShGkrmV0op7pgNzZezLOiDr+IiGSMCn7pWZzhLxXaT5epBKuLduNoT6VDdKXW5YuJYaqFjkJ+gBn+nGXme91qTsytpDqSMzZRVIdfRESyRwW/9Cwu8ieK+c477RYaCv5ah4I/UfhmJccfhOFgO/y5nCI9PTq5UGHfdAYKfkV6REQkg1TwS8/iRbuTpXz7RbvJgj8fd/jbF7a10DHmbxO4bBTBtWCwGf5i3jSHvwfOOeaWq+ycTG/Bbmy8mNccfhFZ4+/uPZH2KYio4JfexV34iVK+bRSlnJjDb2aU8rm2HX7nHEGy4M9I17sWOoqDjPTkFenpRbkWUglCdqY4oSc2WcprLKeIiGSOCn7pWZzDnyx1H+mBKNbTruCPi94xv3NpVhayBoOe0pPLaSxnD84tRxOfdoynX/Arwy8iIlmkgl96Vq35Dn8x33mn3caCP2hdFMUvHsb9ZJ+sZPirwaAz/Nunwz+/UmWx3H4vhm7N+YI/Cx1+FfwiIpJFKvilZ9UgJJ8zSoVc+0hP0FDwd4j0VH1Hf7wQd/izUQQHoRvoxluF/PZZtPsrf/l1fvZdX96UNQv1Dn8GCv7xUp7lit6lERGRbFHBLz2rBiHFvLWNojjnojn8iex7sWBtC/44sz+WsQ5/tPHW4P7LFPNWf7Ez6mbny9x9bI53f+GBvu9rbiVjHf7K5rxzISIisllU8EvPKkFIMZ+j2Gaxabwb77oOf5vOblY7/LUwpDjgDL9z2XmBM0hl/4Lvj/7uPh6YXejrvlYz/IW+z6tfcaTHZWSylIiICKjglz5UfcHfbn58cnOuWKmQp1JrXRAFYWOHPxtd70GP5YzjQllZpDxI5WrAcx63n/FCjhv/+pt97aY8txx11DPR4S/lCV3njeVERESGSQW/9Kxac1Gkp00UJY7ulPKNi3bbTOmJF+1mrMM/8Ay/fzGxHXL85VrIJXsmeN0LHsdXv3eaex+d6/m+MpXh95OlVpTjFxGRDFHBLz1b7fBb6w5/XPD74h1gLJ+jUms9ySReyBkXT1kpgKuhozDADH/BvyjKyvc7SOVayFghz+POnwFWu/S9mFuuMlnKD3SPhG5N+J/ZrTSpx8zyZvaPZvZx//VlZvYVMztsZn9lZiV/fMx/fdhffjBxH2/wx79jZj+eOH6tP3bYzG5MHG/6GCIiMhjpP0PKllUJog21oukyHTr8G5jDX4/0+NuEGclDB+Fgx3IW/bsH22HhbrkWMFbIMT0W5e77GdF5brmaiRn8ABOl6Gd2KxX8wK8C9ya+/j3gbc65xwJngFf7468Gzvjjb/PXw8yuAl4OPAG4Fvgz/yIiD/wp8CLgKuAV/rrtHkNERAZABb/0rNrFot143v76OfxtFu0GjRtvZaPgH3iGP7c9OvxB6KgGjrFCnsmSL/j7mGwzt1LNRH4fEh3+ytYo+M3sYuAlwP/wXxvwPODD/io3Ay/1n1/nv8Zffo2//nXAB51zZefc94DDwNP9x2Hn3APOuQrwQeC6Do8hIiIDoIJfelYNHMWCH8vZoigvN8vwd5jD39jhz8rUmlroBhob2S6LduN/+7FissPfe4E8t1xjx0T6E3oAJvwLmC3U4f8j4NeB+IduL3DWORe/AjsKXOQ/vwg4AuAvP+evXz/ecJtWx9s9hoiIDIAKfulZPcOft5YbKNWLu0JyDn93G2/FU3qy0vEOwsF2+ONIT1a+30Ep+/UbY4UcU2NRR7zfSE/WOvwrW6DgN7OfAE445+5I+1xaMbMbzOx2M7t9dnY27dMREdmyVPBLzyq17hftjjXM4a+2KWobp/Rkp8M/2Ax/vKnXqHf4y/WfidVIz0IfBf/cSoYy/Fsr0vMjwE+a2YNEcZvnAf8d2GVm8VsmFwPH/OfHgEsA/OU7gVPJ4w23aXX8VJvHWMM59y7n3CHn3KH9+/f3/p2KiGxzHQt+M7vJzE6Y2d2JY79tZsfM7E7/8eLEZU2nNcjoqSYX7bYoUsstFu2W23T4a40d/owUwLVgsGM540292r0YGgXl6uqLwHzOmCjmWeojw39uuZqJkZywumh3aQt0+J1zb3DOXeycO0i06PbvnXOvBD4D/LS/2vXAx/znt/iv8Zf/vYt2GLsFeLmf4nMZcCXwVeBrwJV+Ik/JP8Yt/jatHkNERAagmw7/e4kmLzR6m3Puav/xCWg9rWGzTlaypRpEc/iLuTaLdpsU/GOF9mM5s9vhd/Uu/CBsl7Gc9UiPf0E3NVZgoccMfxg6Fsq1zBT8q3P4s1/wt/EbwOvM7DBR3v49/vh7gL3++OuAGwGcc/cAHwK+Bfwf4DXOucBn9F8LfJJoCq97QB8AACAASURBVNCH/HXbPYaIiAxAx5VuzrnPJ+ctd1Cf1gB8z/8yfzrw5Z7PUDJrNcOfw7nmGffmO+122HgrbJjDn5GCPwjdQCM922XRbjLSAzA1lu85wz+/UsO5bOyyC9F55HPGbd87xcuedknnG2SEc+6zwGf95w8Q/d5uvM4K8DMtbv8W4C1Njn8C+EST400fQ0REBqOfduVrzewuH/nZ7Y+1msogI6gShBQLuXqh2mzhbtOddjtM6Yk73Fmb0lMNwgFHeuIIUza+30FJLtoFmCoVei7451b8Lrvj2ZjSMzNe5JeedTl//fVjfPG+k2mfjoiICNB7wf8O4ArgauAR4A82egeavrD11TP8ubgzvb5QbbXxVuhouVlXfD/brcOfz7V+4TRKkhl+gOmxQsc5/EdOL/H6D31j3fSbc8tRwZ+VDj/Arz3/Si7bN8UbPnpXX2sTRERENktPBb9z7rjPaIbAu1l9a7bVVIZm96HpC1tctRZl+Fc3jFpfqJZbRHqAlrGeWuNOuxkp+Aed4d8+YznjRdnJSE/7zPsX7jvJR75+lHsfmVtzfM4X/FnJ8EP0QvV3//k/4cjpZd5263fTPh0REZHeCn4zuyDx5U8B8QSfVtMaZAQld9qF9h3+sfzq2u1486pWsZ74hcPqlJ5sFMC1IKx/r4NQX7Q78hl+v/uy/34nxzpHes4uVwA4cmZ5zfG4w5+VsZyxZ16+l5f8kwv4yNeb9jtERESGqpuxnB8gWnT7eDM7amavBn7fzL5pZncBPwb8e2g9rWFgZy+pqiQW7ULzznSrSE98+2bqU3qK8ZSe9AvgMHSEjoFuvFWPRm2bDr+P9JQ6R3rOLkWF/ZHTS2uOxxn+nZPZKvgBLts3xdmlSmbeoRKR9H3wq9+vNz1EhqmbKT2vaHK45Qi1VtMaZPRUg5BSYTXD33bRbnIsZ6cOf0OkJwsd/sBF5zDIDH8xn53vd5AaM/xTY4WOkZ6zS1GH/+iZtQX/aoc/G4t2k3ZNFgkdzJdrmVpjICLpufGvv8k9D8/x5pc+Me1TkW1GO+1Kz2p+Dn+hXaQnCMjnbE1nvN7hb1nwrx3LmYUpPXHXvZAf3H+ZbbNotz6lJ5Hhr9RwrvW/82qHf22kZ265Rs6ihb9Zs2uyBKy+WBERAbh/diHtU5BtSAW/9CQMHbXQRZGeNot2K7VwzUhO6BzpqTZsvJWFiEv8ImSwHf7tGemZGivgHCy32Z32rO/kH2nS4d8xUcRscP8uvdrlu/rxixUREZG0qOCXnlR9AdzNot1knAdWF2u26vDHmf24IMxChz8+h4Fm+LfNot31kR6AhTYLd+Mu+cNnl9f8PMytVDMbl9k95Qv+ZRX8IiKSLhX80pO4C19a0+FvFulpUvB3iPSs6/BnoOCvDiHSU2yzn8EoKVfXTumZKkX/zu1y/GeXqhRyRjVwPDq3Uj8+t1zN3ISe2M4JRXpERCQbVPBLT6q1uMO/muGvNulMl9tFelp2+P2i3XqHP/2Od3xOg4z0tJt2NErKtZCxQq4ew4k7/K1GczrnOLtc5fHnzwBrJ/WcW85uh3/XpCI9IiKSDSr4pSfxwtJioX2HPy7ukupz+FuO5fSTfTI0tSaO2Qwy0rN9Fu2u/ZmY7lDwL1cDKrWQf3LRTmBtwT+3UmPHRPYW7IIy/CIikh0q+KUncbEezeGPF5u2WLTbUPCPdYr0hNH0n1zOMMvGTrvxi5lBbrzVbi3EKCnXgvouuwCTcaSnxSz+uGC+6sIdmK3dfCvLHf5CPsfMWIEzivSIiEjKVPBLT5IZ/g0v2u0wpScIXb3bXchZJgrgWn3R7uD+y7SbdjRKytXmHf6FFhn+uODfPz3GBTvGOZrs8Gc4ww/RhmDntGhXRERSpoJfehLHTgp5Wy1Um2TtK00iPZ2m9FSDkKK/z3zOMjWlZxhjOavbJMMfizP8Sy0iPfGi112TJS7eM1kfzblSDSjXQnZktMMPsHuypEW7IrJGBqcIyzaggl96Uqmtj/Q0K1R7mdIThK5+n4VcLhMd/voLnAEW/GaWmRc4gxQV/KuRnqlS+7Gc8VjLXZNFLtk9Wd98a27F77Kb4YJ/12SRM8rwi4hIylTwS0+qiYW1bcdy9rjxVj6rHf4BZvgh+n6bTTsaJVGGP9nhXzuWMwwddx87V788jvTsmixyyZ4Jjs+vUK4FzPkXAlnN8EN0bor0iIhI2lTwS0+q9UWsiUW7LSI9G+3w14KwHm+JMvzpF8CrU3oG+1+mmLNtM5YzVsjnGCvkWPKLdv/u3uP8xB9/kcMnou3nzy5HkZjdkyUu2T2Jc3DszDLnlqPr7xjP5pQeUKRHRESyQQW/9KQ+ljNv9bx964238muOxR3/cptIT7xoNysd/vqUngFGeiAqfkd+0W5DpAeiHH8c6XnoVJTRv3/WF/xLVcYKOcaLeS7ZMwlEk3q2Qod/l1+0m4VJUyIisn1ltzUmmVZJzuHv1OFvjPT4r1vNm4/GckbXKWSk4x3Up/QMtuAv5o3qiBeH5WrA2MzYmmNTY/n6HP5HzkU76cbz9s8uVeqbWF2yZwKAB08u8vDZKMuf5Qz/zokioYP5lRo7J7N7niIiMtpU8EtP4p12Sx0W7ZabRHpyOaOQs7aRnnhxbD6fjQ5/tZ7hH+ybYoVcjiADL3AGqVIL18zhh2jhbjyW8/hcVPAf9fP2zy5V2T1ZAuDAzDilfI43ffxbBKHjn16xl0t91z+L4vM+u1xRwS8iIqlRwS89WZPhbzM/vlIL1o3lhCjH37LgT0Z6zAhc+gVwEA5+Sg9sl0W760e1To0V6hn+R85Fhf5qh391c61cznjG5Xs4vVjhdS94HM/7gfOwDM+4i9+ZOLtU5TF7Uz4ZEckEI7u/s2R0qeCXniQz/IV2G281GcsJvuBvEemJFu2uTunJwljOOFY0jEhPFiJMg1Ru8iJwaqzAOb+49fhcGYDvxwX/coXL9k3Vr/vnr37GkM60f3HBr912RUQkTVq0Kz2pZ/jzufqi3aZz+Jtk+CGKAnXT4c9KxCV+0VEcdKQnn8vEVKJBinbaXRvpmR7Ls1CuEYZuTaTHObcm0rPV7PLnrdGcIiKSJhX80pP6HP7Eot2goVCtBSGho3WHv2WG39XHcmamwz+kRbuFnG2PnXaLDR3+UoGlSsDJxTK10HH5/imWqwEnFypRpGeL5t93TaxGekRERNKigl96Uk3utJtrvmg3HrvZquAvt4r0hGF9XUAhb+teSKRhWBn+Yj6XiUXKgxKGjkrQPMO/UK7xqJ/Q8/SDewD47vF5KkHIromt2eGP1x4o0iMiImlSwS89WV20a5iZ78SvLczjDn7TRbv5XP1FQ6Na6OrvGmSlwx9/v0PZaXeE5/DHUbD1c/ijsZzxSM5DvuD/pt9xd/cW7fAX8jlmxgrq8IuISKpU8EtPkhl+aD4vf6kajVmcLK0t7qDTol1X76QXMrLxVnwOhUHvtDvii3bL1eYvAqfGCoQOHjq1CMDTDu4GVgv+XVu04AfYNVVUhl9E6hyj+ztesksFv/Sk2lDwF/O5dZGeJb+R0mRp/TCodot2q0FYn3eflQ7/8DL8o71ot1yLXgQ2y/AD3H9ikULOuGT3JHunStztC/6dWzTSA7BroqRIj4iIpKpjwW9mN5nZCTO7O3Hsv5rZt83sLjP7qJnt8scPmtmymd3pP945yJOX9NSCaJJOfZpOk6z9UqVDh79FwR+EyQ5/NjLttcQY0kEq5Ed70W651irS4wv+2QUO7BgnlzMu3jPJQ6ei0Zy7p7Zwh3+yqEiPiNRpDr+koZsO/3uBaxuO3Qo80Tn3JOC7wBsSl93vnLvaf/zy5pymZE01CNcUv4Vcrr4bbWyx0qbD3y7SE7p6hz+XkQ5/MKQO/6gv2q13+BsiPdNj0QuA+2cXOH/nOACX7J6oX75VF+1CNJpTkR4REUlTx4LfOfd54HTDsU8552r+y9uAiwdwbpJhlcTmWBBn+NcW8MvtOvxt5/CHazL8YQYK4NqQMvyjvmh3pUWGP35ReGapyvk7fMG/Z7J++ZbO8E8UFekRkbovHj7J2z99X9qnIdvMZlQv/wr428TXl5nZP5rZ58zsWZtw/5JB1WDthlqFJotNF33BPzW2sUhPctFuZjL8vggf9JSeYj4b3++g1CM9xeaRHiDR4Y8K/vFijvHi+p+hrWLXZLRoNwsvXEUkG/7w1u+mfQqyzazPWmyAmf0mUAP+0h96BLjUOXfKzH4I+Bsze4Jzbq7JbW8AbgC49NJL+zkNSUG15tZ0+Iv59ZGeZR/pmdhgpKcarEZ6oik96Xe864t2bQiLdke4w9860pMo+Osd/ijSs5XjPBBFepyD+ZXalt1ATEREtraeO/xm9gvATwCvdM45AOdc2Tl3yn9+B3A/8Lhmt3fOvcs5d8g5d2j//v29noakpBqEFAvJDP/6wnyx7Dv8TSI9Y20X7YaZ6/AHoSNn0ZqCQdo+i3YbIz2rPyONHf6tHOeBxG67y4r1iIhIOnoq+M3sWuDXgZ90zi0lju83s7z//HLgSuCBzThRyZZ1Gf4mYzmX/Rz+iSYFfzHfYQ5/Pltz+JPvOgxSMSNTiQZldQ7/2p+J6SaRngt3TWA2AgX/ZLzbrhbuiohIOjpGeszsA8BzgX1mdhR4I9FUnjHgVosiDrf5iTzPBt5kZlUgBH7ZOXe66R3LlrYuw99k0e5iuUYhZ2uuF2s7hz9cfTGRz+UysRFV8l2HQcrn1+9YPEpazuFvEukpFXJcuHOCvVNjwzvBAdg1GUWSzmrhroiIpKRjwe+ce0WTw+9pcd2PAB/p96RkcB6YXeDPb3uI//iSq/qKp1QD19DhXx+9WaoETJTyWJPce6c5/PmM7bRbS5zTIBVz2zPSUyrkKPo403k7Vgv8//7yq+sF81YVd/g1mlNERNKinXa3mU996zj/80sPcnKx3Nf9NM7hL+Zy68ZJLlVq9R1UG5UKOWqhWze5xDkXvZiIM/wZmVpTa3iBMyiF/Kgv2m0e6YGoy793qrTmskMH9/DY86aHdn6DsGeyxNWX7Gq6H0WazGzczL5qZt8ws3vM7D/745eZ2VfM7LCZ/ZWZlfzxMf/1YX/5wcR9vcEf/46Z/Xji+LX+2GEzuzFxvOljiIjIYKjgH0Er1aBl0RjPA++3i1yphWsy7dFOu+s7/M1m8ENU8APrcvzxXeRz2ZvSM4wOfyFv66YdjZJytXmkB2CqVKjn90fJ7qkSf/OaH+EFVx1I+1QalYHnOeeeDFwNXGtmzwR+D3ibc+6xwBng1f76rwbO+ONv89fDzK4CXg48gWiTxj8zs7xfz/WnwIuAq4BX+OvS5jFERGQAVPCPoJ9555f5b59qPuP37GIUK6i2iNN0a/0c/vWLdpcqAZNNZvAD9duWG86j2jDvPmfZ6PAPK8M/8ot2W0R6APZNl3jM3sl1x2UwXGTBf1n0Hw54HvBhf/xm4KX+8+v81/jLr7Eor3cd8EE/pe17wGHg6f7jsHPuAedcBfggcJ2/TavHEBGRAcjWe8zSN+cc3350jot3TzS9fLXD32/B79ZEegq59YtNlyo1JovNf8Tigq/xPOLivpjP2E67iclBg5T3axacc03XPmx1ccHfbCH3H7/iqYw36fzL4Pgu/B3AY4m68fcDZxM7qR8FLvKfXwQcAXDO1czsHLDXH78tcbfJ2xxpOP4Mf5tWjyEiIgOgZ9cRc2apSjVwLJRrTS8/60cDthqJ2a1q41jO3Pqddtt2+ONIT0OHP/D3EUd6MpPhDx2F3BDGcvoXFaO6cLdcCxgr5Jq+mLl07yTn7Ri9SE+WOecC59zVwMVEHfkfSPmU1jCzG8zsdjO7fXZ2Nu3TERHZslTwj5jjcysALQv+TcvwByHFRCyjmG+2aLd1hj9+sdBY8Ff9uwTFjM3hD4aW4Y/+XkZ1NGe5GjaN80i6nHNngc8APwzsMrP4rbmLgWP+82PAJQD+8p3AqeTxhtu0On6qzWM0npc2aBQR2QR65h0x9YJ/pVXB7zP8m9DhL3VatFuutZxM0mrRbnwf+fpOu9E0H7+Zc2qqwXAy/PFjjG6HP2Ss2PxFoAyX3yhxl/98AngBcC9R4f/T/mrXAx/zn9/iv8Zf/vd+l/VbgJf7KT6XEW24+FXga8CVfiJPiWhh7y3+Nq0eQ0REBkAZ/hFzYi4at9msw++cq2/+0/ei3Vpjhr/Jot1qmyk9rTr8/gVAMTGlB6LpPUOI0LcUhMPJ8MfvfGThXY1BiCM9kgkXADf7HH8O+JBz7uNm9i3gg2b2O8A/srrvynuAPzezw8BpogIe59w9ZvYh4FtADXiNcy4AMLPXAp8E8sBNzrl7/H39RovHEBGRAVDBP2LaRXoWyrV6Hn4gGf7GRbvloGOHv3FKT7wOIC6u405/LQzJ59LrDEdjOQdfqNa/3xGdxV+uKdKTFc65u4CnNDn+AFGev/H4CvAzLe7rLcBbmhz/BPCJbh9DREQGQ8+8I+b4/GrB3xiDiRfswiZl+Bt32k3cZzUIqQRh5zn8jQV/Q6Qn7vCn3fGuhWF9M7BBqi/aHdUOfzVsuumWiIiIDI4K/hFz3Ed6nIsWzSbFC3ZhfaG9UdUgrBftsH7RbvzYrQr+1mM540W7fkpPvcOfcsEfDGnRrn8XYXQ7/EHTTbdERERkcPTMO2JO+EgPwGJDrOfMmg7/5s/hT3bhl+sFf4tITz56IbCuwx9HenJrIz1ByotYh5XhL4z8WE5FekRERIZNz7wj5vhcub550XxDwX822eHvo+APQhcVwLlkpCe3JoayWIkee6rTHP4WG28V8msjPWl3+KtDm8M/2mM5KzVFekRERIZNBf8ICULH7EKZy/ZNA+tHc55ZXC34++nwxwuD98+M1Y9FG2+t3mfc4Z9oMYIxfndgfYc/9PcXR3qiP8OUx3IG4XDGcq4u2lWHX0RERDaHnnlHyKnFMkHouGL/FLB+Us/pZKSnjwz/904uAnC5fxyIOvKhg9B34uM40dRYhzn8LRbtFnLZ6vDXgmGN5czG9zsoUYZfHX4REZFhUsE/QuIZ/Ffs9x3+JpGezdjY6YG44PfvJMBqFCXeKXep6jv8Hab0lBsjPfWxnGsX7aad4a8NKdIz8ot2tdOuiIjI0OmZd4TEUZu4874u0rNUrcdw+snwf292kYlingM71kZ6YHV85lI5KvinWizaHWuxaDd+wVDP8OdX5/CnKQiHNKVHi3ZFRERkk+mZd4Qc76LDf54v+PvJ8H/v5AKX7ZvCLDGlJ+7w+0J1yS/a3egc/qDVlJ60Iz1hONSddtN+gTMo0U67ivSIiIgMkwr+EXJ8bgUzuGxf8wz/maUKu6dKFHLWseA/t1Tly/efanrZ904uclkivw+JrL2/305z+Esd5vDH0ZZMZfiHuWh3ZDP8oebwi4gAdx87l/YpyDaiZ94RcmJ+hb1TY0yNFSjlc+sL/sUquydLfpOs9gXlB772ff7le77CSnXt5l2VWsiRM8tcvq+h4G9YbLrUYQ5/Pmfkc7Y+0uPPK168Gk/pSb/D7+rnMkjFeoZ/9Ap+55wfy6lfOyIiyc0wRQZNz7wj5PhcmfN3RpGdqbH8ugz/2aUKuyaLFPPrC+1GZ5YqBKFjbrm65vj3Ty8RhK7+LkIsLlSr9Q5/DTPqewI0U8rn1q0liAv7fMam9ATh2o3GBqX+wmkEF+2W/c+cIj0iIiLDpYJ/hByfW+HAzDgA0+OFNR3+Si1ksRKwe7JEqbC+0G4UL7qda3jREI/kbCz440I1SHT4J4v5NTn/Rs1eeMQvGOIse66e4U+3AK4G4VAW7cYvKqojGOlZLfj1a0dERGSY9Mw7Qo7PrXDeDl/wjxXXFPzxLru7J4tRpKdDhz/eKXd+ZW2H/3snF4BmBf/6RbuTLWbwx0qFfL0IjLXq8Kfd8I52Fh5Ch3+Ex3KWa9GLSGX4RUREhqurZ14zu8nMTpjZ3Ylje8zsVjO7z/+52x83M3u7mR02s7vM7KmDOnlZVQ1CTi5U6qMyZ8YKayI9Z/ymW7vqGf4OBX85LvjXd/j3TJXYNVlac3w1erO6aLfVgt3YWCHXZCxnPId/7ZSetKfW1EJXf1EzSKO8aLdcVaRHREQkDd1WMO8Frm04diPwaefclcCn/dcALwKu9B83AO/o/zSlk9n5aCTnAd/hnxrLr+nwx4uD9kyVKOat46LdeNFtY8H/wOziuu4+JKf0xDvtBi0X7MaaRYviznaxYUpP6ot2g3AoHf76WM4RXLSrSI+IiEg6unrmdc59HjjdcPg64Gb/+c3ASxPH3+citwG7zOyCzThZaS3edCvu8E+PN4/07JosUirkO2b4Vzv8jZGe5gV/faddf7/L1VrHDn+pSbSoHulZ1+FPrwAOQ0foGOrGW2m/ozEI9UiPCn4REZGh6ueZ94Bz7hH/+aPAAf/5RcCRxPWO+mMyQPGmW+fFi3bHCg0d/qhw3z1ZopTvPId/sby+w79QrnFivty8w9+waDfq8Hco+Jt0+OtjOesdfj+WM8WOd+DiUaHDG8s5ijvt1jv8RUV6REREhmlTKhjnnAM2VKGY2Q1mdruZ3T47O7sZp7GtnVte7eADTDeM5TxTX7TbZYa/yaLdB/2EnsYZ/LBamMeF6nIXGf5Skwx/PI0nn8tOhz+O1wy1wz+Ki3arivSIiIikoZ9n3uNxVMf/ecIfPwZckrjexf7YGs65dznnDjnnDu3fv7+P0xCAFV9Mxbn56bEiy9WgXjieXaoyVsgxUcr7KT3tC+g40pMcy/lAXPDvn153/cYoymKlxlSnDH++yaLdho23Gt85SMPq7r/aabcfcaRnGO+UiIiIyKp+nnlvAa73n18PfCxx/FV+Ws8zgXOJ6I8MyLLfEXfCxyWmx6NiO47mnFmssNtP1il2MYd/scmi3Yd8wf+YvZPrrt+4aHe5EjA51r7DXyzkKDcu2g2jeffx/P4sTOmJX2xo0W5/4hdz6vCLiIgMV/sWrGdmHwCeC+wzs6PAG4G3Ah8ys1cDDwEv81f/BPBi4DCwBPziJp+zNLFcWbsgcsbPwF+o1Ng5WeTMUrUe9yl12Gm3GoT1y5ORnpMLZXaMFxhvksFuXLS7WKl1ntLTpMNfC92a6Eze0u/wx4VqfkhjOc1Gc9Fu46ZqIiIiMhxdFfzOuVe0uOiaJtd1wGv6OSnZuJVqwHgxV9+Zdiou+H2H/uxSosPfIcMf77ILazv8Z5aq7JkqNbvJmuhNEDpWqmGXc/iDNcdqgaOYLPgzEHEZZocf4n+fUezwxwX/cP4eRUREJKJW24hYrgb1OA+sRnoWylGH/sxShd1TUYe/U8EfL9gFmC+vdvij+2hR8MeLdkNXjxd1s2i3sbCtBeGaDa7iFxLhNsnwAxRznacobUWr6zP0a0dERGSY9Mw7IpYrDQV/3OH33fqTC5V6d75TB3nJF/yFnK3p8J9OrANotJrhD+u37xTpGS/m6i8OYrXQrSmss9Dhj/P0hSF1pouF3EhO6YlfxJSU4RcRERkqPfOOiOVqwHiioz4zvhrpObVQ5txylYN7o3GapYK1XbQbv0g4b2ZsbaSnXcGfX120G0eCOnX4d02WOLtUwbnVYr4WuDWFdX0Of6od/ngs53D+uxTzOSojHenRrx0REZFh0jPviFhpiPTUM/zlKvedWADgygMzgN/htm2GPyryz985zvxKtV6QRxn+YtPb1BfthiFLlbjgb9/h3ztVohq4NaM/ow7/6o9lFjr88YuNoiI9fYkXaCvDLyIiMlwq+EfEugy/L/jnV2oc9gX/Y8+L5udHc/jbZfijgv38neNUA0e5FrJcCViuBm0y/IkOfz3S077Dv3c6uq9TC+X6sYVydc3t4vsNUpxaExffw9h4C6JIzygW/Mrwi4iIpEPPvCNiuRKsGZcZF/yL5YDDJxaYLOW5cOc4EBeUrTvm8aZb5++YAGBupVrfqXdPy0iPnx8funqHf6rDHP69U2NAtDYgNjtfZv/MWP3rLHX4h5bhz+dGdA6/Ij0iIiJp0DPviFiphmsK/nzOmCzlWShXuX92gSv2T9c3s4oy4uGa7HxSPKXn/J1R4T2/UqsX5btaFPzx/P+TC+V6h3+i2D7SEy8iPrmwWvCfXKisKfjrHf4UC+BafSzncP67FHLt11hsVdUg2lRtWO+UiIiISEQF/4hYqQZMNERopsYKLJSjSM+VPs4D0cZb0LprHnf4D+yI3hGYX6mtdvhbRHrGi3l+7PH7+eBXv18v4Dt1+PdNt+jwT2erwx9PzBnWWM7SiEZ6KkGo/L6IiEgKVPCPiCjDv/afc2aswKPnVnjk3ApXJAr+OFLRarfdRT9lZ7Xgr3JmKZrH32rRLsBrn3clZ5aq3PSl7wGsewHSKN4XIM7wL5ZrLFcD9iU6/GZGztKd0hPUp/Qo0tOPas0pziMiIpICPfuOiMZFuxBtvvWNo+eA1QW7kJio06KLvFSpMVnKs3MiKsjnV2qc8V34VmM5AX7oMbv54cv38sDsIgBTHab0jBXyzIwXOOXve3Y+KvyTHX6IojSpdvjrGX5FevpRDUJKKvhFRESGTs++I2K5snYOP0QLd+O4zJqC3+ftWxWVC+WAyVKhPst/fqXK6cUKZtRfBLTyb5/32PrnjS9Amtk7VaoX/Cd9pz/Z4Yeosx62WG8wDMPeaXdUIz3VIFSHX0REJAV69h0BYRiNzmwssONZ/MW88Zg9k/XjcYa/1aSepUqNqbE8M+OJDv9ShR3jxY5d7h++Yi9PuXQXk6U8uS4K5L3TY5xejAr91h1+SzXiEj+2Ij39qQQhxYIy/CIiIsPWPnMhW8JKLcrcNxb8M77gv2zf1JpCveQ7/K1m8S+WA6ZKhfpoz7mVmt90q3WcJ2Zm/MHPPJlvPzrf1bnvmSpx5PQSALP1Dv/ax8nnLdU5/PWNt4YY6RnNDr8y/CIiKwUUDgAAIABJREFUImlQwT8Clv3c+8ZFstM+kpOM80DnDP9iOerw53PG9FghWrS7WGH3ZPs4T+zy/dNcvn+68xWBfdMl7jxyFoCT82VytjqfP1bIWaoZ/jj6NLQ5/IXcaGb4a8rwi4iIpEHPviNguRoV/OONi3Z9h/6x+5sX/K2KymjRbnTbmfFCfQ5/Nx3+jdozVeL0YoUwdMwulNkzNbYuOpPPWapTeuJpRsMqVksjGulRhl9ERCQdevYdASstCv44w39FQ4e/VO/wt9p4K6i/WIgK/min3VabbvVj79QYQeiYW6kyO19h3/T6x0h7Sk/8wiiOQg3aqEZ6KkE4tHdJpDMzu8TMPmNm3zKze8zsV/3xPWZ2q5nd5//c7Y+bmb3dzA6b2V1m9tTEfV3vr3+fmV2fOP5DZvZNf5u3m9/9r9VjiIjIYKjgHwEr1ag4XJfh95GeK8+bWXO8m0jPpI8HzYwXB9rh3zu9utvu7EJ5zS67sbQ7/NUhd/iLmtIjw1EDXu+cuwp4JvAaM7sKuBH4tHPuSuDT/muAFwFX+o8bgHdAVLwDbwSeATwdeGOigH8H8EuJ213rj7d6DBERGQA9+46AONLTWPBf84MH+JXnXsHjz28s+P2UnpaLdmv1dwdmxgucmC9TroVtZ/D3Ks7rn16scLJhl91YPiMZ/mF1+Ev5XMt3X7ayauCU4c8Q59wjzrmv+8/ngXuBi4DrgJv91W4GXuo/vw54n4vcBuwyswuAHwdudc6dds6dAW4FrvWX7XDO3eacc8D7Gu6r2WOIiMgA6Nl3BKwu2l37z3nRrgl+/dofWJeJj+fwl5t0kZ1zLFYCpsZWO/zxFJ12u+z2Kn7X4ORCuUOHP72Odz3DP6SCv5gfzUhP1OFXpCeLzOwg8BTgK8AB59wj/qJHgQP+84uAI4mbHfXH2h0/2uQ4bR5DREQGQAX/CGi1aLeVeoa/SYe/XAsJQrdm0W7ZX28QHf44s//gqUUqtbBpwZ/2HP5KLcRseBtvFfKjGemp1BTpySIzmwY+Avyac24ueZnvzA/0P1+7xzCzG8zsdjO7fXZ2dpCnISIy0vTsOwJWWkR6Wim2WbS75N8tSC7aje0eQIY/vs/v+Ln9+1pEetLcabfi58f79YYDV/SRHpfi9zwI1SCsv7sk2WBmRaJi/y+dc3/tDx/3cRz8nyf88WPAJYmbX+yPtTt+cZPj7R5jDefcu5xzh5xzh/bv39/bNykiIir4R0GrOfyt1DP8TbrIi+UaQH3R7o7x1RjPIDr8xXyOnRPFesHfssOf8ljOsSF2puOdkNP8ngdBGf5s8RNz3gPc65z7w8RFtwDxpJ3rgY8ljr/KT+t5JnDOx3I+CbzQzHb7xbovBD7pL5szs2f6x3pVw301ewyRbeMjdxzl2NnltE9DtgltvDUCWi3abSXOojebw79YiQr+qSYd/kFM6QHYO1Xi/tkFoHWHP9U5/EEwtPw+UN8VedSm2ijDnzk/Avw88E0zu9Mf+w/AW4EPmdmrgYeAl/nLPgG8GDgMLAG/COCcO21mbwa+5q/3Jufcaf/5rwDvBSaAv/UftHkMkW3jb+58mK89eIYv3fi8tE9FtoGeC34zezzwV4lDlwP/CdhFNIYtDlz+B+fcJ3o+Q+mo5wx/0w5/dF+NBb8Z7JzY/EW7EI3mfODkItCqw5/uRlSVWjjUgr8euao5GMxrrFSM2guYrc4590Wg1Suwa5pc3wGvaXFfNwE3NTl+O/DEJsdPNXsMke3m5EI57VOQbaLngt859x3gagAzyxNlMz9K1PV5m3Puv23KGUpHK5UAMxjrsigttlm0G0d6puI5/GNRkb9rorhu2s9mid85yOeMXU1eVKTe4R/yYtM40lNNcTLRIGjRroiISDo269n3GuB+59xDm3R/sgErtZDxQr7rRaXxwsnmi3bjDP/aDv8g8vuxvT7Gs2+6RK7Ji4pC3qilWPxWA5dapGeUDPvvUURERCKb9ez7cuADia9f67dev0lbpg/eciXoesEurC7abZrhLzdO6Yk67oOY0BPb6++7WZwH0u/wl2vhUBebron0jBBl+EVERNLRdxVjZiXgJ4H/5Q+9A7iCKO7zCPAHLW6n+cqbZLkadL1gF6CYa5Phjzv89Y23htDh9wV/swW7AHlLf6fd4Wb4Ry/SE4aOWugU6REREUnBZjz7vgj4unPuOIBz7rhzLnDOhcC7gac3u5HmK2+e5WrAeLH7f8pczijkrL6DbFJjhz8eyzmIXXZje3yhv79VwZ96hj9Ip8M/QpGe+MWLCn4REZHh24xn31eQiPPEm6l4PwXcvQmPIW2sbDDSA/HmTs0X7eYSC4CnxwvkDPZMNS/GN8O+uMPfItJTyFuqxe+ws+ejGOmJpyxpDr+IiMjw9TWH38ymgBcA/yZx+PfN7GqirdIfbLhMBmCjkR6IYiPNFu0uVmpMlQr1BcD5nPH2VzyFJ1+8a1POtZk90z7D36LDP1kq1DcXS0OlFg5sJGkzoxjpiV+wKcMvIiIyfH0V/M65RWBvw7Gf7+uMZMOWq0E9gtOtUiHfdNHuUjmoz+CP/cSTLuzr/Do5uHeKF1x1gGddua/p5TPjBeZWagM9h3YqqS3aHZ2CP/5ZK2pKj4iIyNBpp90RsFwJWnbHWynlrWlBuVCp1RfsDst4Mc+7X3Wo5eU7xosslGsEoRvYXgDtDH/RbuuxqVtV/L0owy8iIjJ8evYdASvVHjL8heYZ/qVyFOnJknhS0EI5nS7/sDeMGslIj39xqQy/iIjI8OnZdwT0luHPtcjwB0wNucPfyQ6fn59brqby+Kl1+Eco0rOa4devHBERkWHTs+8IWKmGjPdQ8DffeCt7Hf4dvsM/n1KOv1IL61OLhmEUIz0VLdoVERFJjQr+ERDN4d9YwV9qMepyqRIwucEFwIMW7/Y7t5JSh7+W0sZbozSHP87wa9GuiIjI0OnZd4sLQkelFvYY6Wne4Z/OWqTHF/ypdfiDcKid6ZHceCvu8Of0K0dERGTY9Oy7xa1Uo/n0E6WN/VMW87l1O+0uVWqcW64ymbFIT7xoN40MfxA6gtBRyg/vRdAoRnri9QiK9IiIiAyfCv4tbjku+Dfa4S/kqDQUlH/wqe9SroVc+8TzN+38NkO8aHc+hUhP3JlWpKc/msMvIiKSHj37bnHxDrQ9ZfgTHf47j5zlf37pe7zyGZfytIN7NvUc+1Xv8KcQ6SnXUij4C6MY6YleXGosp4iIyPDp2XeLW430bLDgT8zhr9RCfuPDd3HezDg3vugHNv0c+1XM55go5lPp8Ffq8+OHmOHPjWCkR2M5RUREUpOtsLZsWM+RnsSi3b/5x2N85/g8737VofpEnKyZGS8wtzz8Dn9FkZ5NUdVYThERkdSo3bbFxZGefjbeevDUIoWccc0PnLfp57dZdkwUmS+nkOFPIdKTzxlmUBuhgr9SU4dfREQkLXr23eLiDv/4BiM9yY23Ti1U2D1VIpfLbvd1ZryQyljOeod/iFN6zIxibv2i6q2snuHXol0REZGh07PvFrdSjQrS8ULvG2+dWqywd6q06ee2mXaMF1MZy1lJocMPUfRlNCM9+pUjIiIybHr23eJ6XbRbzOfqcZVTi2X2Tme74E+rw19OaX58sZAbqUiPMvwiIiLpUcG/xfUzhz+OWZxerLB3amzTz20z7ZgoMpfmlJ4hd/gLIxbpqajDLyIikho9+25x/SzarQQhzjlOL1TYk/FIz8x4IZU5/HFnemzIBX9p1CI9tejFiwp+ERGR4dOz7xa3umh3Y/+U8Vz5xUrAfLnGvoxHenaMF6nUwnqEaVhW5/APb9EujGakJ58z8hleGC4iIjKqVPBvcSvVgJxtfAfTuNP66LkVAPZOZzzS43fbHXaOvx5FKQy3UC3kbOQ23lJ+X0REJB0q+Le45UrARDGP2caKqTiTHhf82Y/0RBuCDTvHv9rhH/aUntWxqaOgEoSK84iIiKREz8Bb3HI12PCEHkh0+Oeigj/zkZ6JdDv8w160WxrBSM+wXzSJiIhIRM/AW9xyNWB8gwt2YbVj/ei5ZQD2ZHxKT9zhn0+rwz/0KT0jFumpOXX4RUREUqJn4C1upRpseEIPrGbS4w5/1ufw74gjPctD7vAr0rMpqkE49HUQIiIiElHBv8WtVMOeOvzJRbvFvDEzVtjsU9tUM/VFu0Pu8KcY6RmlsZzK8IuIiKSn7yrPzB4E5oEAqDnnDpnZHuCvgIPAg8DLnHNn+n0sWS9etLtRyQz/3qmxDS/6HbYdE9tr0W4hZ9RGKdKjDL+IiEhqNusZ+Mecc1c75w75r28EPu2cuxL4tP9aBmC5GjDew6LdUqLDn/UJPQBTpTw5G/6i3WoQkjMopBDpGaUOfy1Qhl9ERCQtg3oGvg642X9+M/DSAT3Othdl+Df+zxgXXycXKpnP7wOYGTPjReaWh9/hH3acB6KNt0Ypw1/RHH4REZHUbEYl44BPmdkdZnaDP3bAOfeI//xR4EDjjczsBjO73cxun52d3YTT2J4WKzUmSxtPZiWLr71boMMPUY5/2B3+ci2d7HlxBCM9w36XRERERCKbsVLzR51zx8zsPOBWM/t28kLnnDOzdZWLc+5dwLsADh06NDqVzZCdXayy0+fbNyLZtc76LruxHePF4Wf4g5CxNDr8IxbpqQaup7UmIiIi0r++Kxnn3DH/5wngo8DTgeNmdgGA//NEv48j61WDkPlyjd2TG+/QJ7vWWyHDD1GHf27YGf5aOotNiyM2paeqSI+IiEhq+qpkzGzKzGbiz4EXAncDtwDX+6tdD3ysn8eR5s4uRd3u3VP9dfizvstuLJUMf5BShn/ENt6qpBSNkvbM7CYz+7/tnXmcHNV177+nt+mefdFoHYHQwiIJxCIwSzCYxQhDjB0rCbbjhdiGeItt8pyHYz/bSewXFtsxxEtigwkmhCXGNhhsY/aHjREIkIQAjTQSkkbLaPa1Z3q974+qbvUsPdM9W3X1nO/n05/pulXVdW/dqbqnTv3Oua0isj2jrFZEHheRXfbfGrtcROQ2EWkSkW0icnrGPh+xt98lIh/JKD9DRF6z97lN7HRg2Y6hKIqizAxTHYEXAL8Xka3Ai8CjxpjfAjcCl4rILuASe1mZZrrDUQCqp+zhd4mkJzQ1DX9r31De+zhlqBafpCeJ34EHJ2VC/hPYMKIsW5a1y4FV9uda4IdgGe/A14C3Yb3h/VqGAf9D4BMZ+22Y4BiKoijKDDClEdgYs8cYs87+rDHGfNMu7zDGXGyMWWWMucQY0zk91VUy6Up5+Evz9/APC9p1iYe/Muif9MRb2w/2cNY3n+TNw7157edklp7iMviN5uEvQIwx/w8YeX/OlmXtKuCnxuIFoNqWbF4GPG6M6bTnW3kc2GCvqzTGvGCMMcBPR/yWZnJTFEWZJXQEdjFdtod/Mhr+TOPLLVl6KoM++iJxksn8pS7NnWEADnYN5rWfY5Ier4dYwmDZSe5HNfyuIluWtSVAc8Z2B+yy8coPjFE+3jEUpagolnu44n7U4HcxRyU9k/Hwuy9LT0XQjzFWKtJ86bG1//2R/PaNOhW067GM4/gkHm4KEcvg19uN27A98zP6TzjeMTR9s+J27vj9W05XQVEANfhdzVFJzyQ0/LbXOuDzUDaJmXqdoDJkZZGdTKaelMHfl6/B75SH3z5msch6NGjXVWTLsnYQWJqxXYNdNl55wxjl4x1jGMaYHxlj1htj1tfX10+pUYriBA9vPeR0FRQFUIPf1XSFowS8HkonYbCn5BXzygLYiTMKnoqg9SZjMjr+tIc/z4cFxzz83pTBXywefuPIg5MyKbJlWXsY+LCdredsoMeW5TwGvFNEauxg3XcCj9nrekXkbDs7z4dH/JZmclMURZklpmPiLcUhugdiVJf6J2Ww+z2W8VXrkoBdsIJ2AXoHpyLpye9hwbGgXfuBrFg8/KrhL0xE5F7gQmCeiBzAyrZzI/CAiHwM2Af8hb35r4F3AU1AGLgGwBjTKSL/DLxkb/dPGYkaPoWVCSgE/Mb+MM4xFEVRlBlADX4X0xWOTkrOA+DxCH6vUOeSlJxgTbwFTJiLv3coxpd/sZ2v/+nqdHzCZD38MQeDdlPHdzvJpCGeNCrpKUCMMe/PsuriMbY1wKez/M5PgJ+MUb4ZWDtGecdYx1CUYmMiN0cknmRHSy8nLqyclfoocxcdgTPY2z7AH3d3OF2NnOkOxyYVsJvC7/W4JkMPQFXI9vBPIOnZ2tzNr7Ye4qW9XemySWv4nZb0xN0v6YklrYcWNfgVRVFGs+G7z9E5EGUwmnC6KkoRoyNwBt97uonP3vuq09XImal4+AHWNVRz+rHumeAyZfD3TODhT61PpS3NLMt34q6oQxNGpSU9Sfd7+FNxCJqHX1EUZWxO/+fHee8P/uB0NZQiRiU9GbT3R2jvjzim286XrnCMmrLJe/jvvfbsaazNzFNpG/zd4dwM/s6B0QZ/vpKeiNMe/iKQ9MTiKQ+/avgVRVGysaOlj+5wlOopOPIUJRuFb9XOIl22gdjWH3G4JhNjjJlzNwavR6gM+nL28Hf0j2HwT0LSU+Kkhr8YJD32Q4sTb0oURVHcxMfu2ux0FZQiRUfgDDptCciR3iGHazIx/ZE48aShZgoafjdSVerPW9KTTJp0oG++Br9TQbu+IpL0RBOq4VcURcmF3W39TldBKVJ0BM6ga8AyCltdYPCnZC1zycMPUB0KpGcYzkbKuO+w39j0R+OkJqzNR8MfTyRJGmcM1UDaw+9+g181/IqizFnyTJtt3P9SVylQdAS2icaTae/vkd7Cl/SkvNdTCdp1I9Wlfrpz9fDbBn9P+uHIn1ce/pRn2tm0nO6/+8fUw68oiqIojqIjsE2m19gNkp4u24ida5KeylDukp5U0G5quaEmxFAsmXMgbNT2rjvhmS4qSY8G7SqKoiiKo6jBb9M5zOAvfA9/6gFl7kl6/GmPfTbSQbsDVj+mJD5LqkMADOSo43fSw19ckh4N2lUURVEUJ9ER2CblDRaB1r7C9/Cn6jvXPPwpSY8ZR+iYMviHYkkGo4n08pLqUiB3HX/aw6+SnimhGn5FURRFcRYdgW1SAbvL6spcJelJTUY1V6gOBUgkDQPjzEjYE45RGvAClpc/bfDXWB7+vA1+ByU98SKQ9KQ8/D6PSnoURZlb5HvXG8+ZpShTQQ1+m1QQ7AkLKlwj6akM+vDNMa9pVXryrbEz9SSThr5InGV1ZYD1INczQtKTa2rOQpD0RItA0hNVSY+iKIqiOIqOwDapjC4nLKygZzDGUCy7B7kQsGbZnVv6fbDy8EP22Xb7huIYA8fNswz+lIff6xEWVJYA5Jypx0kPf1FJehw8j4qiKG7C/Xd8pVCZsyPwtgPd/M/m5vRyZzhKRYmPBlv20VrgXv7ucHTOpeQEK2gXjgbijqR70HpwSxn8nQNRegZjVIX8VAStfXOV9MQc9PAXl6THGsI0LaeiKHONPNPw0zcUZ0dL78xURpnTzNkR+D+e3cNXfrk9rZfrGohSUxZgQWUQgCMFHrjbFY7OuYBdyPDwZzH4U/KdZWMa/D4gd0lPJO5c/nh/EUl6Ug8tmpZTURRlYt73g+edroJShMxZg39HSy+ReJK2PsuT3xWOUVPqP2rwF3jgbtdAbI56+K02Z5P0ZObc93kkbfBXZhr8LsjSEygiSU/UwQcnRVEUtxFLuv++rxQec3IEHool2NsRBqC5y/rbFU55+C2dd6EH7naHo3MuBz9YaTmBrJNvpcqrS/3UlAXoCkfptT38Ib8Xj+QRtGsbqiUOSnpynSSskEmn5dSgXUVRFEVxhEmPwCKyVESeFpE3ROR1EfmcXf51ETkoIlvsz7umr7q5k0wa/s8vt7P9YM+odU2t/STsJ+jmzkHAkn7UlgaoCvkJ+Dy0FrCHPxpPMhBNzElJT9DvJeDzpLX6I0kb/KEAdWUBOvqPSnpEhPISX+5pOZ3U8NspLONFYfCrh19RFEVRnGQqI3Ac+DtjzGrgbODTIrLaXvevxphT7c+vp1zLSXCoZ5C7X9jHI9sOj1rX2NKX/t7caXv4bQ2/iJXNpZAlPelZdudglh4Yf7bdlMFfFfJTUxrI0PBbcp6KoD/voF0nDFURIeD1EJ0GSc9vt7fQN5RbZqKZ4Oh5VA2/oijKRETjSR5/44jT1VCKjElbMsaYw8aYV+zvfcCbwJLpqthUSXnu93UMjFrXeKSPgM9DXVmA5q4wkXiCgWiCWtuAXlARLGhJT2rSrbno4QdLrjOepCfg9RD0e6gtD9CREbQLUF7iyz8tp0NSFL9XpizpOdwzyN/818s8sPnANNUqf6Lq4VcUZY7y6v7uSe33iZ9uTjskFWU6mJYRWESWAacBm+yiz4jINhH5iYjUTMcx8iWlzd/XMfqC2dHSx8r6co6tK6W5czAdAJrShy+oChZ0lp7UJGFzMWgXLO99tqDdXjtAV0SoLQ1wsGuQpDk6YVd50Je3ht+p/PE+r2fKkp4DXdaD7/4xHnxni1hc03IqiqLky4fu2DTxRoqSI1MegUWkHHgQ+Lwxphf4IbACOBU4DHw7y37XishmEdnc1tY21WqM4oD9ZLy/MzxqqurGll5OXFjB0tpSmrvCdNqTbtWWHvXwF3Ie/rSkZ456+KtCgXHTcqbkO7VlgbR3eZiHP0dJT8RxD//UJT2Hui2Dv9k2/J0glkji9Qhej0p6FEVRcmXvGA5LRZksU7JkRMSPZezfY4z5OYAx5ogxJmGMSQI/Bs4aa19jzI+MMeuNMevr6+unUo0xSRk4/ZE4HQNHAzy7w1GO9EY4YWEFS2tKOdwzlE7NmZq5dkFlCf2ReM6e4Nkm9daiobrU4Zo4Q3WpP+vEW5nynbryo29Aqux0nuVBH3059msqu4wTWXoAAtMi6bHeVDn5ajiWSKp+X1EUZRLoJFzKdDGVLD0C3AG8aYz5Tkb5oozN3gtsn3z1Jk9zZ5iUQzFT1rPDDtg9YWEFDTUhEknD64esCyqt4bdz8c92ph5jDFubJ9b7NbX2M6+8JD0J1VzDkvRkz9KTMvgzJU+psspg7h5+p/PHT4ek57Dt4T/QNTjqTddsEU0kVc6jKIoyCTZ897miSM+sOM9URuHzgA8BF41IwXmziLwmItuAdwBfmI6KjoUxhq6BsQ2/5q4w65ZWA7C/86h+OZWh58SFlSyttTzkrx20jOyURGa+Q7n4n9nZxlXf/8OERv+u1n5WzS+fpVoVHtUhPwPRxJg3wWEe/rLRBr8VtJtrWs6Eo1IUK2h3ipIe28M/GEvQ3j/2tTLTxBJJx+IgFEVR3M5f/+dLDBSo4kBxD1PJ0vN7Y4wYY07JTMFpjPmQMeZku/zdxpjReTGniVsea+TcG59K59RPMRRLcKQ3wrkr6hAZ7eGvCvlZUFnC0hrL4N/abOXqT3mEF9oe/j3t/TNV9TF57YBVj8y0oSMxxrC7tZ+Vc9ngH2fyrZ5whoc/0+AvTRn8fsLRxKj/mbGIxp01VC0N/9Q8O4e6B9NtSAWyzzaxuFEPv6IoyiR5blc7n/nvV9JvnRVlMrh6FF5eX85gLMFbIwzzg7aMYUV9OYsqg8MM/saWXk5YWIGIsKg6iEes7SuCvrRRsqyujOMXlHPX83tJzuIU1ylDf0979owqrX0R+iLxOW3wV9oG/chMPcmkoS8SH9/DH7QCenOR9UTjSUdnh/VPh6SnZ4hT7TddTun4Y4kkfp9q+BVFUSbL041tHP+V3/DCng6nq6K4FFcb/GuXVAKkNfgpUobN0tpSjq0rS+fiN8aw80g/Jy6sACyDalFVCDiq3wfweIRPv2MlO4/087tZnPwiFZwz8gEmk6ZWa91cNvir7TcxPSNm2+0bimPM0QeClIff6xHKAl4AKkosg78vh1z80YSznumpSnqGYgk6B6KceZyVGfeAQ5l6VMOvKIoyPVz9oxecroLiUlw9Cq+oLyfg87D9YM+w8lSGnqU1pRxbV8p++wFgR0sf/ZE4axdXpbddWmsZ/CNz2l95ymKOm1fG957eNSvBjkOxRDoF11vjePh3HbHeAsxpgz80XNLT0R8Ztpzy5vu9HiqCPqrsvPyQ4eHPQQ8ZjScdy9ADU5f0pDL0LJ9XzrzywKx4+AejCcLR4edWNfyKoijTRy6SVEUZiatHYb/Xw0kLK0Z5+A90hgn4PMyvKOGYulLa+6P0R+L8ZnsLInDRSfPT26Z0/CNnrfV6hE9esILtB3t5Zuf0zxMwkqbWfhJJw5LqEHs7wlkv6Ka2fiqCPuZXlMx4nQqVqgxJz5bmbs785hM8u7ONbtvjn1oPlqwnc7m8JA9JT8Ldkp5Uhp5F1UEaakpnRcP/2Xtf4axvPsl3Ht9J75D1ABZz+E2JoihKMfH5+7ewo6V3lKY/nkiOcoAqSgrXj8KrF1ex/WDPMC98c1eYhuoQHo+wrK4MgH0dAzy2vYUzl9Uyr/yosZzK1JMZ4JniPactYUl1iE/f8wrn/MuTXHjL0zw+QxKflH5/w9qFROPJ9IRJI2myA3ZTHuu5SCpotzsc47837SNp4KFXD47y8IMl1arMWK4IpiQ9uXj4Ew4H7U5N0pPK0LO4KmRNMtc5s5KeaDzJc7vaKQ14ue3JXZx/09P84Jkm+oZimodfURRlmvjV1kNs+O5zXHHbcwxGE+ny7zy+kyv/7fe8cUhz9yujcb3Bv3ZJJb1D8WH65ObOQRpsQ/4Y++8zjW00Hunj8rULh+2fkvTUlo42+AM+DzdvPIUrT1nE+avmEfR7+dQ9L/N0Y+u0t6PxSB8Bn4eLTrTePmST9TS1DrCyfu7KeQAqgn5ErAw0j2w7jAg88eYROuy0k5nzE3zh0uP5wiWrMvbN3cMfSxhHg039Xs+U8i+nPPwLq4IsrQlxqHtwRl8Fv3awh0g8yT9dtYZHPvsnnH5MNTf/tpEQaUxeAAAWBElEQVSX9nbhUw+/oihzjGzzxUwXu1r7Oemrv6WjP8KyGx7lB8/sBuDZWVAlKO7D9aPwGluP//qho6+xmrvCLK2xDPlj6yyD/z+f3wvAZWtGGPw12T38AOetnMfNG9dx88Z13H/dORy/oIK/uftlnm9qn9Z27GjpY2V9eTq//lgGf3c4Snt/ZE7r98GSW1WU+HjwlQOEowk+deEKeofi/HZ7CzDcw3/+qnouPOGohKu8xFqXq4bf6bScUzH4D/UMUlcWIOj3srS2lHjScLhn5rz8L77VCcCZy2pZu6SKO685iwc/eQ7vOKGeC46f/tm0FUVRCplwhvd9Jnlpb9ew5Zt+u4OWniF+93oLPeGJE1QocwPXG/wnLqzA65G0jr9vKEZ3OJaW6lQE/dSVBWjri7CuoYrF1aFh+y+vLyfo93DcvLIJj1UV8nP3x97GsXWlfPKeV2jrm76JuRpbejlxYQX1FSWUBbxjGvypDD2rFsxtgx+sTD1d4Rgr55fz2YtWURbw8sSbltwq0+AfibvSck5R0tM9xKJqa06J1IPtTMp6Xnyrg5Xzy6nLkMydcWwtd15zFp9+x8oZO66iKEohMtZcMbPF2f/yJNfe/TKftvP3H8wiE1bmDq43+IN+Lyvry9OBKimDJmXgABxje/k3rF00av/asgB/vOFiNozw/GejtizADz54BoPRBF//1etTrT5gee6P9EbS8wMcV182Zi7+dErO+oppOa6bSen4/3L9UoJ+L+84cT7xpMHvFUJ+b9b9Sv1eRHLT8EcSSQK+7L8100xZ0tMzmE47m5KuzVTgbiJp2Ly3izOX1c7I7yuKoriN7zy+0+kqsLdjgBse3MZ5Nz7FFbc9p5N3zWFcb/ADrFlSmfbwpwyahpqjnvxja1MG/9hGfU1ZAI8nd6225VVeyaPbDk9LEO8OO2D3BHt+gOPmlY+Zi7+ptZ8Sn4clNaFR6+YaVSE/Po/w3tOXAHC5/TBXFQqMG9Ds8QjlAR99Q+N7Xpo7w7T1Djkq6fFNWcM/xOIqy8O/uDqER6wMVjPBm4d76YvEedtxavAriqJA4aTPfMqOO3z9UC/Hf+U3BVMvZXYpDoN/cRWtfRFa+4bSXvCUpAfgfWc0cN3bl+ck28mV6y5YwYkLK/jKL1+bsi46laHnxIXWRGLHzSvjQNcgkfhR/V8yaXj9UC/L68vx5vFwUqxsPKOBL152Qjrj0oUn1FPi81AV8k24b3nQl1XS0x+J89WHtnPRt5+hfSDKn64b/VZotghMQdLTNxSjLxJnkS1hS00y1zxDk2+l9PtnqcGvKIrNwe7BWZnHplAphLaPVYUV//Brx2ZeV5yjKAz+tYstQ/kTP32ZWx5rZHl92bC8+uevqudL7zppWo8Z8Hm46X2n0BWOceEtz/CNR95ITwA1Fu3jrNvR0kdVyM+CSst4XT6vDGNgf0cYYwxPvnmEK//t9/xxTwfnLK+b1na4latOXcJ1F6xIL5eV+Ljq1MWsXVI1zl4WFUHfmEG74Wica+58kXs27efP1y/l2S9eyFWnLpnWeufDVCQ9qUm3FtkefrDees3UTf6lvZ001IRGxcgoijI32X6wh/NufIq7X9jndFXmNNkeOs6/+Wm+/3QTD205OMs1UpyiKAz+1YsrCXg9NB3p47MXreQXnzpvVvLUr1tazZPXX8CVpyzmJ394iytu+z37O4YbVC/v6+IDP36B9d94gu89tWvUb0TiCZ7f3c5JiyrSdU69idjdNsDNjzXysbs20x+J869/uY4vXzG9Dy7FxE3vO4Vbrz5twu3qK0p4blc733+6iQHb8B+KJfj4XZt5eV8Xt159Kv/3vSen9e9OMRVJT2oeh0wDfOX8crY0d/OPv3p9WgPOjTG8+FanevcVRUmTSjzx1YemJ9bNjTjv3x+/Drc81sjn7tvCUGx2sgkpzjKx/sEFVAT9PPSZ85hfUTIsQ8hssLS2lG//xTquOW8Zf3XHJj5w+wvcf905dIejfPt3O3lqRyt1ZQHOXVHHt363k6Dfy8fPX57e//tPNbGvI8w/vntNumyZbfB/63eNNLX28/6zjuGfrlqjs5VOQK4Pef981Vq++eib3PJYIz98ZjdVIT+DsQRd4Sjf2riOK09ZPMM1zY2Az0MsYWjtG2J+RXDYusM9g3zh/i382WkN/MWZS0ftO5aH//pLjyeeMPz0j/u478VmvvXn67jiFEuytOtIH//rZ9t4+6p5fPz85eNmOgIrDe6tT+zi9UO9JI2hYyCq+n0lb0RkA3Ar4AVuN8bc6HCVlGliV+voOLS5Rmvv9DlWJktqLBiPuGr65wRFYfADnLSo0tHjr11Sxd1//TY+8OMXeNetz9EzGKMy6OOLl53AR89dRonPw+fu28I3Hn2TwWiCT7x9OXs7BvjBM7t572lLhuWKrwr5mVceoKm1n41nNPDN96zNK6hYGZ/l9eXc8dEzeWV/Fz97+UA6a8HFJ87n8pOd0+yP5LI1C7j9uT381e2buO/ac6i154po7Rvigz/exJ72AV7Y04nHI2w8o4HDPYP81wv76A7H2H6oFxFYUHnU4K8rL+Gmjadw3QXL+eLPtvG5+17F7xVWLajgA7dvYjCaYGtzN3c9v5cNaxdmfcA80jvEE2+2Uhn0cfFJC/B6hKDfw4Y1hXPulMJHRLzA94FLgQPASyLysDHmDWdrpkyGZ3e2scaW19aVBbjtydFvtOcSdz2/lzcOz86Mtz9/5cCsHGciYokk1z+wlY1nNBCOxBERAj5hXnkJJy2qdKXTMiWJGs+h+NSOI/i9HsLRBOeuqKMiOL7DzCmkEIJK1q9fbzZv3ux0NaaFl/d18Q8/f43L1izgYyM8pbFEks/ft4VHXzvMoqogpQEvXeEYT1x/QdqYS/G1h7YTTST5xntO1iDdOczzu9u55s6XWFFfzucvWYUBvv27Rg50DXL7h9fzw2d384emdq44ZTGPvd5CImnS8SunLq3h9o+sH/N3+4Zi/NUdL/LmoV6qSv0kk4b7rzubaNzwr0/s5NX9XWPuB1ZswcYzGnJ6E6BMDyLysjFm7M50KSJyDvB1Y8xl9vKXAIwx/zLW9pMdJ7Yf7OFAlzXLdDyZ5BevHuSZxjY++LZj6B6M8ei2w+ltb736VEqGzb0hdl0zgx8NxlhxWXXlJYhV9/Q2xhjiScP+zjALK4MMRONs2tPJ5ScvJJE09A3Fqa8osX/PkDSQNNZvJu2DhPzevGWpTo3lz+/uSE9sORkmmm9kSXUonUN+eX0Ze9oGOKWhim0HrFTcX7jkeBZVB9nfEaYrHEXEmkNl+8Fe9neGSRozagKsEp+Hc1bU8UxjG/UVJaxZXMlLb3Vy+rE1LKwMsn5ZDcbA3o4wlSEfv9p6mNWLKtnS3MXbj6+nrS/CmsVVBP0efrnlEGctq6Eq5OfJHa28ur+bgM/jqvSXN/7ZycOMVBFrDgGvCH6fEPL7iCaSxBPJ9KzpHgFBrL8CINyzaR/P7co+KelN7zuZ6tIA3eEo//vB19Lll65ewMYzGhiIxLn+ga3j/k/87UUrWbukil2t/bywp2PY8f5+wwnsbR+gpTfCJSfNp7zEx/UPbMXnEeJJQ8jvZbAA5Uvnrqgj4PNw7oo63nPqEuZXBifeaQS5jBFq8DvAH5raueWxRrY0d3Pb+0/j3esKQ0KiFCbP7mzjE3dtJmrr+Ut8Hu685kzOXTGPwWiCj975Ii/t7eR9pzfwtxevGpahajx6wjE+cPsLHOga5N5PnM3qxc6+JVOyU6QG/0ZggzHm4/byh4C3GWM+k7HNtcC1AMccc8wZ+/blHwD6dw9s5cEC8YAqyki2fPVSdrf18+zOdu54bg8DszQ7r1KY3PnRM3nHifMn3nAEavAXMMYYWnqHHA8MVdxBa+8QrXag7YLKIPUVR2NVYokkXQPRSXkFIvEEQ9EkVaXqqS9k5qrBn8lkx4nmzjC9QzH8Xg9ejyDAawd7rIkOEToHovQMRqmvKKE0YKlcM4dFY3v0Uw53QdKeeJ9XbK++5Z33iODxgEeEI71DlAZ8GGN5+1fUl+MR4VDPIIuqgvg8HkSsba2/1q8bY4i4yDucSBqe2tFKLJFkzeIqDIaWniEqg372dQ6wuDqEVywPa2nAS99QnMqQj76hOEGfl0XVQXoGYxzpjRDyewn6PTR3DnL8gnJaeodYvagSEWEgGqetL0LXgNVXSWPNTXPFyYvoj8QJ+j1pz3oqKUG3La3d2xHG5xGW1pbS3BlmQWWQuvIALT1D+Lwe5pUF2N3WT2XIT8jvpa48gNfjobV3iKqQn+7BGEG/l/6hOAsqSzjcM5R2rLzV3k91KEBVqZ/W3gjt/REqQ36CPg+dA1HOXTGPY+pKMcYwGEuk/8fyxRgz7K1PLJEkkTR4PUI0nmQgEifg85A01nupnsEYrX0RvB5hMJrAYDjYNYjP66Eq5Ke61JrLZiCSYEFlCYmMf/p4whBLJOm125tIGjr6o5T4PFQE/QR8Yv/PW9dHMmn9FYT2/gilAS+xhMEjVmpWKy10kFDAm76Wmlr7GYwm8HjE7mPr2C09Q9SWBRiMJogkksTiScpLfBi7TcfY5z2WsPp5IBqnKuSnvS/KiYsq6A7H6BmMcWxdKV6PsK8jjDHW9VXi9zAYTabvB82dYeLJJOuPreVAV5hE0lAe9NMdjhJLWBN59kfi+DxCXyROdShAeYmXEp81gWfvUJyecJQV88s50DVI71AMY6yMeIe6hzimtpQSn4doIslb7dabqfa+COFYgnllR8/5uoZq9nYMcPKSKhZWBSclfVKDX1EUpQgoUoN/ViQ9iqIoxU4uY4T7IigURVGUYuAlYJWIHCciAeBq4GGH66QoilKUFE2WHkVRFMU9GGPiIvIZ4DGstJw/McbM3aTtiqIoM4ga/IqiKIojGGN+Dfza6XooiqIUOyrpURRFURRFUZQiZsYMfhHZICKNItIkIjfM1HEURVEURVEURcnOjBj8GTMoXg6sBt4vIqtn4liKoiiKoiiKomRnpjz8ZwFNxpg9xpgocB9w1QwdS1EURVEURVGULMyUwb8EaM5YPmCXKYqiKIqiKIoyizgWtCsi14rIZhHZ3NbW5lQ1FEVRFEVRFKWomSmD/yCwNGO5wS5LY4z5kTFmvTFmfX19/QxVQ1EURVEURVHmNmKMmf4fFfEBO4GLsQz9l4APZJtURUTagH2TPNw8oH2S+xYS2o7CoRjaANqOQmMq7TjWGDOnPSM6TuSFtre40fYWN5Np74RjxIxMvJXvDIpTGchEZLMxZv1k9y8UtB2FQzG0AbQdhUaxtMMpdJzIHW1vcaPtLW5mqr0zNtOuzqCoKIqiKIqiKM6jM+0qiqIoiqIoShFTDAb/j5yuwDSh7SgciqENoO0oNIqlHW5krp17bW9xo+0tbmakvTMStKsoiqIoiqIoSmFQDB5+RVEURVEURVGy4GqDX0Q2iEijiDSJyA1O1ycXRGSpiDwtIm+IyOsi8jm7vFZEHheRXfbfGqfrmgsi4hWRV0XkEXv5OBHZZPfJ/SIScLqOEyEi1SLyMxHZISJvisg5buwPEfmC/T+1XUTuFZGgG/pDRH4iIq0isj2jbMzzLxa32e3ZJiKnO1fzo2Rpwy32/9Q2EfmFiFRnrPuS3YZGEbnMmVrPDdw4Towk33FjvOtERD5ib79LRD7iVJtyIdfxRURK7OUme/2yjN9wxbWWzzhUDP2bz3jlxv6drnEtW3+KyBki8pq9z20iIhNWyhjjyg9Wus/dwHIgAGwFVjtdrxzqvQg43f5egTVfwWrgZuAGu/wG4Can65pje64H/ht4xF5+ALja/v7vwCedrmMObbgL+Lj9PQBUu60/gCXAW0Aoox8+6ob+AN4OnA5szygb8/wD7wJ+AwhwNrDJ6fqP04Z3Aj77+00ZbVht369KgOPs+5jX6TYU48et48QY7chr3Mh2nQC1wB77b439vcbp9o3T7pzGF+BTwL/b368G7re/u+Zay2cccnv/5jteubF/s4wJ09afwIv2tmLve/mEdXL6pEzhZJ4DPJax/CXgS07XaxLteAi4FGgEFtlli4BGp+uWQ90bgCeBi4BH7H+8do4aOcP6qBA/QJV945ER5a7qD/sG2mzfGHx2f1zmlv4Alo24MY55/oH/AN4/1nZOf0a2YcS69wL32N+H3auw5is5x+n6F+OnWMaJMdo17riR7ToB3g/8R0b5sO0K6ZPP+JJ5Ddn3v3Z7e1dca/mOQ27v33zHK7f271THtWz9aa/bkVE+bLtsHzdLelL/MCkO2GWuwX4tdRqwCVhgjDlsr2oBFjhUrXz4LvD3QNJergO6jTFxe9kNfXIc0Abcab86vl1EynBZfxhjDgLfAvYDh4Ee4GXc1x8psp1/t173f43lhQH3tsGNFN25znHcyNZuN52PfMaXdLvs9T329m5pb77jkKv7dxLjldv7N8V09ecS+/vI8nFxs8HvakSkHHgQ+LwxpjdznbEe2Qo6fZKIXAm0GmNedrouU8SH9drth8aY04ABrFdtaVzSHzXAVVgDx2KgDNjgaKWmCTec//EQkS8DceAep+uiuBu3jxu5UkTjS64UxTiUK8U8XuWKE/3pZoP/ILA0Y7nBLit4RMSPddO+xxjzc7v4iIgsstcvAlqdql+OnAe8W0T2AvdhvXa9FagWkdQMzm7okwPAAWPMJnv5Z1g3Xrf1xyXAW8aYNmNMDPg5Vh+5rT9SZDv/rrruReSjwJXAB+0bPLisDS6naM51nuNGtna75XzkO76k22WvrwI6cE978x2H3N6/+Y5Xbu/fFNPVnwft7yPLx8XNBv9LwCo7qjuAFcjxsMN1mhA7kvoO4E1jzHcyVj0MpCKwP4Kl0SxYjDFfMsY0GGOWYZ37p4wxHwSeBjbam7mhHS1As4icYBddDLyBy/oD69Xo2SJSav+Ppdrhqv7IINv5fxj4sJ3V4GygJ+MVaUEhIhuwJAnvNsaEM1Y9DFxtZ544DliFFYClTD+uHCdGMolxI9t18hjwThGpsb2s77TLCopJjC+Z52Gjvb3BJdfaJMYhV/cv+Y9Xru7fDKalP+11vSJytn3+PkwuY7vTQQ1T+WBFNu/Eisz+stP1ybHOf4L1GmcbsMX+vAtLj/YksAt4Aqh1uq55tOlCjmZRWI51wTUB/wOUOF2/HOp/KrDZ7pNfYkXDu64/gH8EdgDbgbuxMhcUfH8A92LpOGNYnq6PZTv/WIFa37ev+deA9U7Xf5w2NGHpL1PX+b9nbP9luw2N5JBdQT9T6hvXjRNjtCGvcWO86wQrnqTJ/lzjdNtyaPuE4wsQtJeb7PXLM/Z3xbWWzzhUDP2bz3jlxv6drnEtW38C6+1ztxv4HiMCvsf66Ey7iqIoiqIoilLEuFnSoyiKoiiKoijKBKjBryiKoiiKoihFjBr8iqIoiqIoilLEqMGvKIqiKIqiKEWMGvyKoiiKoiiKUsSowa8oiqIoiqIoRYwa/IqiKIqiKIpSxKjBryiKoiiKoihFzP8HPY1ezy6U8l4AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 1440x360 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "num_frames = 10000\n",
    "batch_size = 32\n",
    "gamma      = 0.99\n",
    "\n",
    "losses = []\n",
    "all_rewards = []\n",
    "episode_reward = 0\n",
    "\n",
    "state = env.reset()\n",
    "for frame_idx in range(1, num_frames + 1):\n",
    "    epsilon = epsilon_by_frame(frame_idx)\n",
    "    action = model.act(state, epsilon)\n",
    "    next_state, reward, done, _ = env.step(action)\n",
    "    replay_buffer.push(state, action, reward, next_state, done)\n",
    "    \n",
    "    state = next_state\n",
    "    episode_reward += reward\n",
    "    \n",
    "    if done:\n",
    "        state = env.reset()\n",
    "        all_rewards.append(episode_reward)\n",
    "        episode_reward = 0\n",
    "        \n",
    "    if len(replay_buffer) > batch_size:\n",
    "        loss = compute_td_loss(batch_size)\n",
    "        losses.append(loss.data.item())\n",
    "        \n",
    "    if frame_idx % 200 == 0:\n",
    "        plot(frame_idx, num_frames, all_rewards, losses)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<p><hr></p>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
